diff --git a/checks/symbol_compat.py b/checks/symbol_compat.py new file mode 100644 index 000000000..db703d8bb --- /dev/null +++ b/checks/symbol_compat.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +"""Cross-version KPM symbol compatibility gate. + +Usage: symbol_compat.py [extra.kpm dirs...] + +Fails when a symbol referenced by any known .kpm (the freshly built demos in +kpms/*/ plus previous-generation .kpm files passed as extra paths) could not +resolve on either backend. + +The kpimg export set is read from kernel/kpimg.elf (build it first). Provider +names come from each KPM's own .kpm.export section, which is what the loader +actually consults -- not from its global symbol table, so an ordinary global +helper cannot make an unresolved import pass. + +Exit status: + 0 every referenced symbol resolves on both backends + 1 at least one symbol is unresolvable + 2 the gate could not run (no kpimg.elf, or no .kpm inputs found) +""" +import re +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +NM = 'aarch64-linux-gnu-nm' +SECTION = '.kpm.export' + + +def require_tool(name): + if shutil.which(name) is None: + sys.exit(f'required tool not found in PATH: {name}') + + +def kpimg_exports(): + """Names the kpimg runtime publishes to KPMs (kernel/kpimg.elf).""" + require_tool(NM) + require_tool('readelf') + elf = ROOT / 'kernel' / 'kpimg.elf' + if not elf.exists(): + sys.exit('kernel/kpimg.elf missing; build it first') + out = subprocess.run([NM, str(elf)], capture_output=True, text=True).stdout + names = set() + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[2].startswith('__kp_symbol_'): + names.add(parts[2][len('__kp_symbol_'):]) + if not names: + sys.exit('no __kp_symbol_* entries in kernel/kpimg.elf; is it built?') + return names + + +def lkm_exports(): + """Names the LKM backend publishes (lkm/kpm/symbols.c).""" + src = ROOT / 'lkm' / 'kpm' / 'symbols.c' + if not src.exists(): + sys.exit('lkm/kpm/symbols.c missing') + text = src.read_text() + lits = re.findall(r'\{\s*"([^"]+)"\s*,', text) + kfuncs = ['kf_' + m for m in re.findall(r'KP_KPM_KFUNC_ENTRY\((\w+)\)', text)] + names = set(lits) | set(kfuncs) + names.discard('kf_name') # macro parameter artifact, not a real symbol + return names + + +def export_names(kpm): + """Return only identifiers emitted by KPM_EXPORT in this object file. + + KPM_EXPORT(foo) creates a local section object named + __kpm_export_foo. Ordinary GLOBAL functions are not exports, and a + function explicitly exported by the macro may itself be static. + """ + out = subprocess.run(['readelf', '-sW', str(kpm)], capture_output=True, text=True) + if out.returncode: + raise RuntimeError(f'readelf failed for {kpm}: {out.stderr.strip()}') + prefix = '__kpm_export_' + return { + fields[7][len(prefix):] + for line in out.stdout.splitlines() + if len(fields := line.split()) >= 8 + and fields[0].rstrip(':').isdigit() + and fields[6] != 'UND' + and fields[7].startswith(prefix) + } + + +def elf_syms(path, want): + out = subprocess.run(['readelf', '-sW', str(path)], capture_output=True, text=True).stdout + syms = set() + for line in out.splitlines(): + c = line.split() + if len(c) < 8 or not c[0].rstrip(':').isdigit(): + continue + bind, ndx, name = c[4], c[6], c[7] + if want == 'und' and ndx == 'UND' and name not in ('', 'UND'): + syms.add(name) + elif want == 'global' and bind == 'GLOBAL' and ndx != 'UND': + syms.add(name) + return syms + + +def classify(sym, table): + if sym in table: + return 'compat-table' + if sym.startswith(('kf_', 'kv_')): + return 'slot+kernel' + if sym in ('printk', 'kallsyms_lookup_name', 'kallsyms_on_each_symbol'): + return 'slot+kernel' + return 'KERNEL-ONLY' + + +def collect_kpms(): + kpms = {} + for p in sorted((ROOT / 'kpms').glob('*/*.kpm')): + kpms[p] = 'current' + for arg in sys.argv[1:]: + path = Path(arg) + candidates = sorted(path.glob('*.kpm')) if path.is_dir() else [path] + for p in candidates: + if p.is_file(): + kpms.setdefault(p, 'previous-generation') + return kpms + + +def main(): + kpimg, lkm = kpimg_exports(), lkm_exports() + + kpms = collect_kpms() + if not kpms: + # A clean checkout has kpimg.elf but no built .kpm files. Reporting + # PASS here would make the gate succeed without checking anything. + print('FAIL: no .kpm inputs found (build the demos, or pass a ' + 'directory containing .kpm files)', file=sys.stderr) + return 2 + + providers = set() + for p in kpms: + providers |= export_names(p) + + rows, failures = {}, [] + for p, gen in kpms.items(): + for sym in sorted(elf_syms(p, 'und')): + k, l = classify(sym, kpimg), classify(sym, lkm) + if k == 'KERNEL-ONLY' and l == 'KERNEL-ONLY' and sym in providers: + k = l = 'module-export' # provided by another KPM + rows.setdefault(sym, {})[p.name] = (gen, k, l) + if 'KERNEL-ONLY' in (k, l): + failures.append((sym, gen, p.name, k, l)) + + print(f"kpimg exports: {len(kpimg)} LKM table: {len(lkm)} " + f"KPMs checked: {len(kpms)} KPM providers: {len(providers)}\n") + print(f"{'symbol':36} {'kpimg':14} {'lkm':14} {'gen':20} used by") + for sym in sorted(rows): + for mod, (gen, k, l) in sorted(rows[sym].items()): + print(f"{sym:36} {k:14} {l:14} {gen:20} {mod}") + + if failures: + print('\nFAIL: symbols unresolvable without a matching kernel symbol:') + for sym, gen, mod, k, l in failures: + print(f" {sym} ({gen}, {mod}): kpimg={k} lkm={l}") + return 1 + print('\nPASS: every referenced symbol resolves on both backends') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/checks/test_kpm.c b/checks/test_kpm.c new file mode 100644 index 000000000..a4e447b58 --- /dev/null +++ b/checks/test_kpm.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Host unit tests for the KPM loader: link arena, symbol resolution, and + * cross-module dependency policy. Build: + * gcc -O2 -Wall -Wextra -o /tmp/test_kpm checks/test_kpm.c && /tmp/test_kpm + */ +#include +#include +#include +#include +#include "../kernel/include/kpmlink.h" +#include "../kernel/include/kpmsymbol.h" +#include "../kernel/include/kpmdep.h" + +struct fixture { + unsigned char image[2048] __attribute__((aligned(16))); + struct kpm_link_state link; + unsigned long known_slot; + unsigned long zero_slot; +}; + +static unsigned long compat(const char *name, void *context) +{ + struct fixture *f = context; + if (!strcmp(name, "kf_known")) return (unsigned long)&f->known_slot; + if (!strcmp(name, "kf_late")) return (unsigned long)&f->zero_slot; + if (!strcmp(name, "compat_api")) return 0x1000; + return 0; +} + +static unsigned long kernel_function(const char *name, void *context) +{ + (void)context; + if (!strcmp(name, "late") || !strcmp(name, "new_function")) return 0xffff000001000000UL; + if (!strcmp(name, "printk")) return 0xffff000002000000UL; + return 0; +} + +static unsigned long kernel_data(const char *name, void *context) +{ + (void)context; + return !strcmp(name, "kernel_data") ? 0xffff000003000000UL : 0; +} + +static unsigned long slot(unsigned long value, void *context) +{ + struct fixture *f = context; + return kpm_link_pointer(&f->link, f->image, value); +} + +static void test_resolver(void) +{ + struct fixture f = { .known_slot = 0xffff000004000000UL }; + unsigned int size = 256; + struct kpm_symbol_resolution r; + assert(!kpm_link_reserve(&f.link, &size, 8)); +#define RESOLVE(n) kpm_symbol_resolve(n, compat, kernel_function, kernel_data, slot, &f, &r) + /* Compatibility table hit with a populated pointer slot. */ + assert(!RESOLVE("kf_known")); + assert(r.kind == KPM_SYMBOL_FUNCTION_POINTER && *(unsigned long *)r.address == f.known_slot); + assert(r.address != (unsigned long)&f.known_slot); + /* Compatibility slot present but NULL -> fall back to the kernel. */ + assert(!RESOLVE("kf_late")); + assert(*(unsigned long *)r.address == 0xffff000001000000UL); + /* No slot at all: a fresh one is allocated and populated. */ + assert(!RESOLVE("kf_new_function")); + assert(r.target == 0xffff000001000000UL); + /* Plain (non-pointer) kernel symbol resolves to its own address. */ + assert(!RESOLVE("new_function")); + assert(r.address == 0xffff000001000000UL && r.kind == KPM_SYMBOL_DIRECT); + /* kv_* data pointer slot. */ + assert(!RESOLVE("kernel_data")); + assert(r.address == 0xffff000003000000UL); + assert(!RESOLVE("kv_kernel_data")); + assert(*(unsigned long *)r.address == 0xffff000003000000UL && r.kind == KPM_SYMBOL_DATA_POINTER); + /* printk is a pointer variable in the KPM headers. */ + assert(!RESOLVE("printk")); + assert(*(unsigned long *)r.address == 0xffff000002000000UL); + /* Non-pointer compatibility entry. */ + assert(!RESOLVE("compat_api") && r.address == 0x1000); + /* Unresolvable names. */ + assert(RESOLVE("missing") == -1 && !r.address); + assert(RESOLVE("") == -1); + assert(RESOLVE(NULL) == -1); + assert(RESOLVE("k") == -1); + assert(RESOLVE("kf") == -1); + assert(RESOLVE("kf_") == -1); + /* No slot allocator: must report arena exhaustion, not success. */ + assert(kpm_symbol_resolve("kf_new_function", compat, kernel_function, kernel_data, + NULL, &f, &r) == -2); +#undef RESOLVE +} + +static void test_link(void) +{ + unsigned char image[256] __attribute__((aligned(16))) = { 0 }; + unsigned int size = 17; + struct kpm_link_state state; + unsigned long plt; + struct kpm_link_slot *s; + unsigned long base = 0x10000000UL; + + assert(sizeof(struct kpm_link_slot) == KPM_LINK_SLOT_SIZE); + assert(!kpm_link_reserve(&state, &size, 2)); + assert(state.offset == 32 && size == 96); + plt = kpm_link_plt(&state, image, 0xffff000001234000UL); + s = (void *)plt; + assert(s->insn[0] == 0xd503245f && s->insn[1] == 0x58000070); + assert(s->insn[2] == 0xd61f0200 && s->target == 0xffff000001234000UL); + assert(kpm_link_pointer(&state, image, s->target) == (unsigned long)&s->target); + assert(state.used == 1); + /* Slots are deduplicated by target. */ + assert(kpm_link_plt(&state, image, 0x4000)); + assert(!kpm_link_plt(&state, image, 0x5000)); + /* Capacity and image-size bounds. */ + size = KPM_LINK_MAX_IMAGE; + assert(kpm_link_reserve(&state, &size, 1) == -1); + size = 16; + assert(kpm_link_reserve(&state, &size, ~0UL) == -1); + /* Branch range check (+-128MB, 4-byte aligned). */ + assert(kpm_link_branch_in_range(base, base - (1UL << 27))); + assert(kpm_link_branch_in_range(base, base + (1UL << 27) - 4)); + assert(!kpm_link_branch_in_range(base, base + (1UL << 27))); + assert(!kpm_link_branch_in_range(base, base - (1UL << 27) - 4)); + assert(!kpm_link_branch_in_range(base, base + 1)); +} + +static int put_calls; + +static void put_cb(void *provider, void *ctx) +{ + (void)provider; + (*(int *)ctx)++; +} + +static void test_deps(void) +{ + void *deps[4]; + unsigned int count = 0; + int a = 0, b = 0; + void *pa = &a, *pb = &b; + + /* Recording is idempotent per provider. */ + assert(kpm_dep_record(deps, &count, 4, pa) == 1); + assert(kpm_dep_record(deps, &count, 4, pa) == 0); + assert(kpm_dep_record(deps, &count, 4, pb) == 1); + assert(count == 2); + assert(kpm_dep_record(deps, &count, 4, (void *)1) == 1); + assert(kpm_dep_record(deps, &count, 4, (void *)2) == 1); + assert(count == 4); + /* Full table must be reported, never silently accepted. */ + assert(kpm_dep_record(deps, &count, 4, (void *)3) == -1); + + /* Forgetting an edge invokes the callback exactly once. */ + put_calls = 0; + assert(kpm_dep_forget(deps, &count, pa, put_cb, &put_calls) == 1); + assert(put_calls == 1); + assert(kpm_dep_forget(deps, &count, pa, put_cb, &put_calls) == 0); + assert(put_calls == 1); + assert(count == 3); + assert(kpm_dep_forget(deps, &count, pb, NULL, NULL) == 1); + assert(count == 2); + /* Remaining entries are the two synthetic providers, order-independent. */ + assert((deps[0] == (void *)1 || deps[0] == (void *)2) && + (deps[1] == (void *)1 || deps[1] == (void *)2) && deps[0] != deps[1]); + assert(kpm_dep_record(NULL, &count, 4, pa) == -1); + assert(kpm_dep_record(deps, &count, 0, pa) == -1); +} + +int main(void) +{ + test_link(); + test_resolver(); + test_deps(); + puts("PASS: link arena, symbol resolution, dependency policy"); + return 0; +} diff --git a/doc/en/module.md b/doc/en/module.md index 4fc2e8f39..1e57ace1e 100644 --- a/doc/en/module.md +++ b/doc/en/module.md @@ -318,6 +318,19 @@ static long my_init(const char *args, const char *event, void *reserved) } ``` +## Cross-KPM Symbol Exports (LKM style) + +KPMs can publish symbols to later KPMs the way an LKM uses `EXPORT_SYMBOL`: + +```c +int my_helper(int a, int b); +KPM_EXPORT(my_helper); +``` + +- `KPM_EXPORT` places the symbol in the `.kpm.export` section; later KPMs reference it with a plain `extern`, the loader resolves it by name and records the dependency. +- A module whose exports are still imported refuses to unload (`-EBUSY`), matching `rmmod` in-use semantics; importers release their providers on unload, and a failed load rolls references back. +- Behaviour is identical on the kpimg and LKM backends. + ## Loading and Managing Modules Use the SuperCall API from userspace to manage KPMs at runtime: diff --git a/doc/zh-CN/module.md b/doc/zh-CN/module.md index 04fc91535..b2ab26ef2 100644 --- a/doc/zh-CN/module.md +++ b/doc/zh-CN/module.md @@ -316,6 +316,19 @@ static long my_init(const char *args, const char *event, void *reserved) } ``` +## 跨模块符号导出（LKM 风格） + +KPM 支持类似内核模块 `EXPORT_SYMBOL` 的跨模块符号导出： + +```c +int my_helper(int a, int b); +KPM_EXPORT(my_helper); +``` + +- `KPM_EXPORT` 将符号写入 `.kpm.export` 段；后加载的 KPM 直接 `extern` 引用即可，加载器按名解析并记录模块依赖。 +- 仍被其他 KPM 引用的模块拒绝卸载（返回 `-EBUSY`），与 `rmmod` 的 in-use 语义一致；引用方卸载或加载失败时自动回滚引用计数。 +- kpimg 与 LKM 两个后端行为一致。 + ## 从用户空间加载和管理模块 使用 `supercall.h` 中的 API 在运行时管理 KPM： diff --git a/kernel/include/kpmdep.h b/kernel/include/kpmdep.h new file mode 100644 index 000000000..f3dff3cb1 --- /dev/null +++ b/kernel/include/kpmdep.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _KP_KPMDEP_H_ +#define _KP_KPMDEP_H_ + +/* LKM-style dependency tracking: an importer records its providers; a provider + * counts importing modules. Unload order mirrors the kernel module loader: + * a provider with live importers refuses (-EBUSY), an importer releases its + * providers when it goes away, and a failed load rolls its edges back. */ + +#define KPM_DEP_MAX 16 + +/* 1 = new edge recorded, 0 = edge already present, -1 = table full. */ +static inline int kpm_dep_record(void **deps, unsigned int *count, unsigned int cap, void *provider) +{ + unsigned int i; + if (!deps || !count || !provider || cap > KPM_DEP_MAX) return -1; + for (i = 0; i < *count; i++) { + if (deps[i] == provider) return 0; + } + if (*count >= cap) return -1; + deps[(*count)++] = provider; + return 1; +} + +/* Drops one recorded edge, invoking put(provider, ctx) exactly once for it. + * 1 = dropped, 0 = no such edge. */ +static inline int kpm_dep_forget(void **deps, unsigned int *count, void *provider, + void (*put)(void *provider, void *ctx), void *ctx) +{ + unsigned int i; + if (!deps || !count) return 0; + for (i = 0; i < *count; i++) { + if (deps[i] == provider) { + deps[i] = deps[*count - 1]; + (*count)--; + if (put) put(provider, ctx); + return 1; + } + } + return 0; +} + +#endif diff --git a/kernel/include/kpmlink.h b/kernel/include/kpmlink.h new file mode 100644 index 000000000..fd45db23e --- /dev/null +++ b/kernel/include/kpmlink.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _KP_KPM_LINK_H_ +#define _KP_KPM_LINK_H_ + +/* Kept inside each KPM allocation; no persistent kpimg/preset ABI changes. */ +#define KPM_LINK_SLOT_SIZE 32U +#define KPM_LINK_MAX_IMAGE (64U * 1024U * 1024U) +#define KPM_RELOC_ADR_GOT_PAGE 311U +#define KPM_RELOC_LD64_GOT_LO12_NC 312U +#define KPM_RELOC_GOT_LD_PREL19 309U + +struct kpm_link_slot { + unsigned int insn[4]; + unsigned long target; + unsigned long reserved; +}; + +struct kpm_link_state { + unsigned long offset; + unsigned int capacity; + unsigned int used; +}; + +static inline int kpm_link_reserve(struct kpm_link_state *state, unsigned int *size, + unsigned long count) +{ + unsigned long offset = ((unsigned long)*size + 15UL) & ~15UL; + if (offset > KPM_LINK_MAX_IMAGE || count > (KPM_LINK_MAX_IMAGE - offset) / KPM_LINK_SLOT_SIZE) + return -1; + state->offset = offset; + state->capacity = count; + state->used = 0; + *size = offset + count * KPM_LINK_SLOT_SIZE; + return 0; +} + +static inline struct kpm_link_slot *kpm_link_get(struct kpm_link_state *state, void *image, + unsigned long target) +{ + struct kpm_link_slot *slots = (void *)((char *)image + state->offset); + unsigned int i; + for (i = 0; i < state->used; i++) { + if (slots[i].target == target) return &slots[i]; + } + if (state->used == state->capacity) return 0; + struct kpm_link_slot *slot = &slots[state->used++]; + slot->insn[0] = 0xd503245fU; /* bti c */ + slot->insn[1] = 0x58000070U; /* ldr x16, .+12 (target at +16) */ + slot->insn[2] = 0xd61f0200U; /* br x16: tail call preserves x30 */ + slot->insn[3] = 0xd503201fU; /* nop */ + slot->target = target; + slot->reserved = 0; + return slot; +} + +static inline unsigned long kpm_link_plt(struct kpm_link_state *state, void *image, + unsigned long target) +{ + struct kpm_link_slot *slot = kpm_link_get(state, image, target); + return (unsigned long)slot; +} + +static inline unsigned long kpm_link_pointer(struct kpm_link_state *state, void *image, + unsigned long target) +{ + struct kpm_link_slot *slot = kpm_link_get(state, image, target); + return slot ? (unsigned long)&slot->target : 0; +} + +static inline int kpm_link_branch_in_range(unsigned long place, unsigned long target) +{ + long delta = target - place; + return !(delta & 3L) && delta >= -(1L << 27) && delta < (1L << 27); +} + +#endif diff --git a/kernel/include/kpmodule.h b/kernel/include/kpmodule.h index 82ed1da2e..b4c9a158f 100644 --- a/kernel/include/kpmodule.h +++ b/kernel/include/kpmodule.h @@ -47,4 +47,20 @@ typedef long (*mod_eventcall_t)(const char *event, const char *args, void *reser #define KPM_EVENT(fn) \ static mod_eventcall_t __kpm_eventcall_##fn __attribute__((__used__)) __attribute__((__section__(".kpm.event"))) = fn +/* LKM EXPORT_SYMBOL equivalent: publish a symbol for later KPMs. The entry + * lands in the allocatable .kpm.export section; the loader relocates it and + * resolves other KPMs' `extern` references against it, tracking module + * dependencies the same way the kernel module loader does. */ +struct kpm_export_entry { + const char *name; + void *target; +}; + +#define KPM_EXPORT(sym) \ + static const struct kpm_export_entry __kpm_export_##sym __attribute__((__used__)) \ + __attribute__((__section__(".kpm.export"), aligned(8))) = { \ + .name = #sym, \ + .target = (void *)&sym, \ + } + #endif diff --git a/kernel/include/kpmsymbol.h b/kernel/include/kpmsymbol.h new file mode 100644 index 000000000..b86d9d657 --- /dev/null +++ b/kernel/include/kpmsymbol.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _KP_KPM_SYMBOL_RESOLVE_H_ +#define _KP_KPM_SYMBOL_RESOLVE_H_ + +/* Resolver callbacks are supplied by kpimg or LKM. A pointer slot is not code. */ +#define KPM_SYMBOL_DIRECT 1U +#define KPM_SYMBOL_FUNCTION_POINTER 2U +#define KPM_SYMBOL_DATA_POINTER 3U + +struct kpm_symbol_resolution { + unsigned long address; + unsigned long target; + unsigned int kind; +}; + +typedef unsigned long (*kpm_symbol_lookup_fn)(const char *name, void *context); +typedef unsigned long (*kpm_symbol_slot_fn)(unsigned long target, void *context); + +static inline int kpm_symbol_prefix(const char *name, char a, char b) +{ + return name && name[0] == a && name[1] == b && name[2] == '_' && name[3]; +} + +static inline int kpm_symbol_is_function_pointer(const char *name) +{ + if (kpm_symbol_prefix(name, 'k', 'f')) return 1; + const char *special[] = { "printk", "kallsyms_lookup_name", "kallsyms_on_each_symbol" }; + unsigned int i; + for (i = 0; i < sizeof(special) / sizeof(special[0]); i++) { + const char *a = name, *b = special[i]; + if (!a) continue; + while (*a && *a == *b) { a++; b++; } + if (!*a && !*b) return 1; + } + return 0; +} + +static inline int kpm_symbol_resolve(const char *name, kpm_symbol_lookup_fn compatibility, + kpm_symbol_lookup_fn kernel_function, + kpm_symbol_lookup_fn kernel_data, + kpm_symbol_slot_fn slot, void *context, + struct kpm_symbol_resolution *result) +{ + unsigned long address = 0, target = 0; + int function_pointer = kpm_symbol_is_function_pointer(name); + int data_pointer = kpm_symbol_prefix(name, 'k', 'v'); + result->address = result->target = 0; + result->kind = KPM_SYMBOL_DIRECT; + if (!name || !name[0]) return -1; + if (compatibility) address = compatibility(name, context); + if (address && (function_pointer || data_pointer)) { + target = *(const unsigned long *)address; + if (target) { + if (slot) address = slot(target, context); + if (!address) return -2; + result->address = address; + result->target = target; + result->kind = function_pointer ? KPM_SYMBOL_FUNCTION_POINTER : KPM_SYMBOL_DATA_POINTER; + return 0; + } + /* A null predeclared kf_* slot is not a successfully resolved function. */ + address = 0; + } + if (address) { + result->address = result->target = address; + return 0; + } + if (function_pointer || data_pointer) { + const char *base = kpm_symbol_prefix(name, 'k', 'f') || data_pointer ? name + 3 : name; + kpm_symbol_lookup_fn lookup = data_pointer ? kernel_data : kernel_function; + if (lookup) target = lookup(base, context); + if (!target) return -1; + address = slot ? slot(target, context) : 0; + if (!address) return -2; + result->address = address; + result->target = target; + result->kind = function_pointer ? KPM_SYMBOL_FUNCTION_POINTER : KPM_SYMBOL_DATA_POINTER; + return 0; + } + /* Exact symbols preserve data identity; compiler-suffixed functions are fallback only. */ + if (kernel_data) address = kernel_data(name, context); + if (!address && kernel_function) address = kernel_function(name, context); + if (!address) return -1; + result->address = result->target = address; + return 0; +} + +#endif diff --git a/kernel/patch/include/module.h b/kernel/patch/include/module.h index 1e50d4194..37da3534b 100644 --- a/kernel/patch/include/module.h +++ b/kernel/patch/include/module.h @@ -8,6 +8,8 @@ #include #include +#include +#include struct load_info { @@ -51,6 +53,14 @@ struct module unsigned int ro_size; void *start; + struct kpm_link_state link; + + /* LKM-style cross-KPM exports (KPM_EXPORT -> .kpm.export) + deps */ + const struct kpm_export_entry *exports; + unsigned int export_count; + struct module *deps[KPM_DEP_MAX]; /* providers we import from */ + unsigned int dep_count; + unsigned int export_refs; /* modules importing from us */ struct list_head list; }; diff --git a/kernel/patch/module/module.c b/kernel/patch/module/module.c index acd7bc0f7..fce840002 100644 --- a/kernel/patch/module/module.c +++ b/kernel/patch/module/module.c @@ -26,6 +26,8 @@ #include "module.h" #include "relo.h" +#include +#include #define SZ_128M 0x08000000 @@ -198,6 +200,94 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs, unsigned return true; } + +extern struct module modules; +static spinlock_t module_lock; + +/* LKM loader semantics: search the export tables of already-loaded KPMs + * (.kpm.export / KPM_EXPORT) for @name and record the dependency edge on + * @importer. Runtime-table symbols win, exactly like vmlinux exports shadow + * module exports in the kernel's find_symbol(). */ +static unsigned long kpm_module_export_lookup(const char *name, struct module *importer) +{ + struct module *pos; + unsigned long found = 0; + unsigned long flags; + + if (!importer) return 0; + flags = kp_private_spin_lock(&module_lock); + list_for_each_entry(pos, &modules.list, list) { + unsigned int i; + for (i = 0; i < pos->export_count; i++) { + if (!strcmp(name, pos->exports[i].name)) { + { + int edge = kpm_dep_record((void **)importer->deps, &importer->dep_count, + KPM_DEP_MAX, pos); + if (edge < 0) { + /* Table full: fail the resolution so the KPM load aborts. + * Silently succeeding here would allow the provider to be + * unloaded while the consumer still holds its symbols. */ + logke("dependency table full; cannot import %s from %s\n", name, pos->info.name); + goto out; + } + /* Pin the provider before returning its address. The + * reference stays provisional through importer init and is + * rolled back if loading fails. */ + if (edge > 0) pos->export_refs++; + } + found = (unsigned long)pos->exports[i].target; + goto out; + } + } + } +out: + kp_private_spin_unlock(&module_lock, flags); + return found; +} + +static unsigned long kpm_compat_lookup(const char *name, void *context) +{ + unsigned long addr = symbol_lookup_name(name); + if (addr) return addr; + return kpm_module_export_lookup(name, context); +} + +static unsigned long kpm_function_lookup(const char *name, void *context) +{ + (void)context; + return kallsyms_lookup_name ? kallsyms_lookup_name_by_suffix(name) : 0; +} + +static unsigned long kpm_data_lookup(const char *name, void *context) +{ + (void)context; + return kallsyms_lookup_name ? kallsyms_lookup_name(name) : 0; +} + +static unsigned long kpm_pointer_slot(unsigned long target, void *context) +{ + struct module *mod = context; + return kpm_link_pointer(&mod->link, mod->start, target); +} + +/* Reserve pointer slots and a worst-case PLT/GOT entry per allocated RELA. */ +static int kpm_prepare_link(struct module *mod, const struct load_info *info) +{ + unsigned long count = info->sechdrs[info->index.sym].sh_size / sizeof(Elf_Sym); + unsigned int i; + if (count > KPM_LINK_MAX_IMAGE / KPM_LINK_SLOT_SIZE) return -E2BIG; + for (i = 1; i < info->hdr->e_shnum; i++) { + const Elf_Shdr *section = &info->sechdrs[i]; + if (section->sh_type != SHT_RELA || section->sh_info >= info->hdr->e_shnum || + !(info->sechdrs[section->sh_info].sh_flags & SHF_ALLOC)) continue; + if (section->sh_size % sizeof(Elf64_Rela)) return -ENOEXEC; + if (section->sh_size / sizeof(Elf64_Rela) > KPM_LINK_MAX_IMAGE / KPM_LINK_SLOT_SIZE - count) + return -E2BIG; + count += section->sh_size / sizeof(Elf64_Rela); + } + return kpm_link_reserve(&mod->link, &mod->size, count) ? -E2BIG : 0; +} + /* Change all symbols so that st_value encodes the pointer directly. */ static int simplify_symbols(struct module *mod, struct load_info *info) { @@ -211,27 +301,38 @@ static int simplify_symbols(struct module *mod, struct load_info *info) const char *name = info->strtab + sym[i].st_name; switch (sym[i].st_shndx) { case SHN_COMMON: - if (!strncmp(name, "__gnu_lto", 9)) { - logkd("Please compile with -fno-common\n"); - ret = -ENOEXEC; - } + set_load_error(info, "COMMON symbol unsupported: compile with -fno-common and without LTO"); + ret = -ENOEXEC; break; case SHN_ABS: break; - case SHN_UNDEF: - unsigned long addr = symbol_lookup_name(name); - // kernel symbol cause overflow in relocation - // if (!addr) addr = kallsyms_lookup_name(name); - if (!addr) { - logke("unknown symbol: %s\n", name); + case SHN_UNDEF: { + struct kpm_symbol_resolution resolved; + int rc = kpm_symbol_resolve(name, kpm_compat_lookup, kpm_function_lookup, + kpm_data_lookup, kpm_pointer_slot, mod, &resolved); + if (rc) { + if (rc == -1 && ELF_ST_BIND(sym[i].st_info) == STB_WEAK) { + sym[i].st_value = 0; + break; + } + logke("unresolved symbol: %s (%s)\n", name, + rc == -2 ? "link arena exhausted" : "unavailable on this kernel"); if (!info->info.error_msg[0]) - snprintf(info->info.error_msg, sizeof(info->info.error_msg), "unknown symbol: %s", name); - ret = -ENOENT; + snprintf(info->info.error_msg, sizeof(info->info.error_msg), + "unresolved symbol: %s (%s)", name, + rc == -2 ? "link arena exhausted" : "unavailable on this kernel"); + ret = rc == -2 ? -ENOMEM : -ENOENT; break; } - sym[i].st_value = addr; + sym[i].st_value = resolved.address; break; + } default: + if (sym[i].st_shndx >= info->hdr->e_shnum) { + set_load_error(info, "invalid symbol section index"); + ret = -ENOEXEC; + break; + } secbase = info->sechdrs[sym[i].st_shndx].sh_addr; sym[i].st_value += secbase; break; @@ -298,9 +399,38 @@ static int rewrite_section_headers(struct load_info *info) info->sechdrs[0].sh_addr = 0; for (int i = 1; i < info->hdr->e_shnum; i++) { Elf_Shdr *shdr = &info->sechdrs[i]; - if (shdr->sh_type != SHT_NOBITS && info->len < shdr->sh_offset + shdr->sh_size) { + Elf_Shdr *strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (shdr->sh_name >= strings->sh_size || + !memchr(info->secstrings + shdr->sh_name, 0, strings->sh_size - shdr->sh_name)) + return -ENOEXEC; + if (shdr->sh_addralign && (shdr->sh_addralign & (shdr->sh_addralign - 1))) return -ENOEXEC; + /* Subtraction, not sh_offset + sh_size: the sum can wrap for a + * crafted 64-bit offset and let the bounds check pass. */ + if (shdr->sh_type != SHT_NOBITS && + (shdr->sh_offset > info->len || shdr->sh_size > info->len - shdr->sh_offset)) { + return -ENOEXEC; + } + /* sh_name indexes the section string table; bound it before + * move_module() hands the name to strcmp(). */ + { + const Elf_Shdr *section_strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (shdr->sh_name >= section_strings->sh_size || + !memchr(info->secstrings + shdr->sh_name, 0, + section_strings->sh_size - shdr->sh_name)) + return -ENOEXEC; } + /* sh_name indexes the section string table; bound it before any + * find_sec()/move_module() string operation consumes it. */ + { + const Elf_Shdr *shstr = &info->sechdrs[info->hdr->e_shstrndx]; + if (shdr->sh_name >= shstr->sh_size || + !memchr(info->secstrings + shdr->sh_name, 0, + shstr->sh_size - shdr->sh_name)) + return -ENOEXEC; + } + if (shdr->sh_addralign && (shdr->sh_addralign & (shdr->sh_addralign - 1))) + return -ENOEXEC; /* Mark all sections sh_addr with their address in the temporary image. */ shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset; } @@ -343,6 +473,11 @@ static int move_module(struct module *mod, struct load_info *info) if (!mod->event && !strcmp(".kpm.event", sname)) mod->event = (mod_eventcall_t *)dest; if (!mod->info.base && !strcmp(".kpm.info", sname)) mod->info.base = (const char *)dest; + + if (!mod->exports && !strcmp(".kpm.export", sname)) { + mod->exports = (const struct kpm_export_entry *)dest; + mod->export_count = shdr->sh_size / sizeof(struct kpm_export_entry); + } } mod->info.name = info->info.name - info->info.base + mod->info.base; mod->info.version = info->info.version - info->info.base + mod->info.base; @@ -358,6 +493,16 @@ static int setup_load_info(struct load_info *info) { int rc = 0; info->sechdrs = (void *)info->hdr + info->hdr->e_shoff; + if (!info->hdr->e_shstrndx || info->hdr->e_shstrndx >= info->hdr->e_shnum) { + set_load_error(info, "invalid section string table index"); + return -ENOEXEC; + } + Elf_Shdr *section_strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (section_strings->sh_type != SHT_STRTAB || section_strings->sh_offset > info->len || + section_strings->sh_size > info->len - section_strings->sh_offset) { + set_load_error(info, "invalid section string table bounds"); + return -ENOEXEC; + } info->secstrings = (void *)info->hdr + info->sechdrs[info->hdr->e_shstrndx].sh_offset; if ((rc = rewrite_section_headers(info))) { @@ -415,6 +560,10 @@ static int setup_load_info(struct load_info *info) if (info->sechdrs[i].sh_type == SHT_SYMTAB) { info->index.sym = i; info->index.str = info->sechdrs[i].sh_link; + if (info->index.str >= info->hdr->e_shnum) { + set_load_error(info, "invalid symbol string section index"); + return -ENOEXEC; + } info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset; break; } @@ -425,6 +574,31 @@ static int setup_load_info(struct load_info *info) set_load_error(info, "module has no symbols (stripped?)"); return -ENOEXEC; } + Elf_Shdr *symbols = &info->sechdrs[info->index.sym]; + if (symbols->sh_size % sizeof(Elf_Sym) || symbols->sh_link >= info->hdr->e_shnum) { + set_load_error(info, "invalid ELF symbol table"); + return -ENOEXEC; + } + Elf_Shdr *strings = &info->sechdrs[symbols->sh_link]; + if (strings->sh_type != SHT_STRTAB || !strings->sh_size) { + set_load_error(info, "invalid ELF symbol strings"); + return -ENOEXEC; + } + Elf_Sym *table = (void *)info->hdr + symbols->sh_offset; + for (unsigned int i = 0; i < symbols->sh_size / sizeof(Elf_Sym); i++) { + if (table[i].st_name >= strings->sh_size || + !memchr(info->strtab + table[i].st_name, 0, strings->sh_size - table[i].st_name)) { + set_load_error(info, "unterminated ELF symbol name"); + return -ENOEXEC; + } + } + { + int export_sec = find_sec(info, ".kpm.export"); + if (export_sec && info->sechdrs[export_sec].sh_size % sizeof(struct kpm_export_entry)) { + set_load_error(info, "malformed .kpm.export section"); + return -ENOEXEC; + } + } return 0; } @@ -447,7 +621,6 @@ static int elf_header_check(struct load_info *info) } struct module modules = { 0 }; -static spinlock_t module_lock; long load_module_ex(const void *data, int len, const char *args, const char *event, const char *source, void *__user reserved) @@ -459,7 +632,10 @@ long load_module_ex(const void *data, int len, const char *args, const char *eve if ((rc = elf_header_check(info))) goto out; if ((rc = setup_load_info(info))) goto out; - if (find_module(info->info.name)) { + rcu_read_lock(); + bool module_exists = find_module(info->info.name) != NULL; + rcu_read_unlock(); + if (module_exists) { logkfd("%s exist\n", info->info.name); set_load_error(info, "module already exists"); rc = -EEXIST; @@ -488,6 +664,10 @@ long load_module_ex(const void *data, int len, const char *args, const char *eve layout_sections(mod, info); layout_symtab(mod, info); + if ((rc = kpm_prepare_link(mod, info))) { + set_load_error(info, "module link arena too large or invalid"); + goto free; + } if ((rc = move_module(mod, info))) { set_load_error(info, "allocate executable module memory failed"); @@ -504,16 +684,41 @@ long load_module_ex(const void *data, int len, const char *args, const char *eve rc = (*mod->init)(mod->args, event, reserved); if (!rc) { - logkfi("[%s] succeed with [%s] \n", mod->info.name, args); - list_add_tail(&mod->list, &modules.list); + logkfi("[%s] initialized\n", mod->info.name); + { + unsigned long flags; + bool duplicate; + rcu_read_lock(); + flags = kp_private_spin_lock(&module_lock); + duplicate = find_module(mod->info.name) != NULL; + if (!duplicate) + list_add_tail_rcu(&mod->list, &modules.list); + kp_private_spin_unlock(&module_lock, flags); + rcu_read_unlock(); + if (duplicate) { + set_load_error(info, "module already exists"); + rc = -EEXIST; + (*mod->exit)(reserved); + goto free; + } + } goto out; } else { set_load_error(info, "module init failed"); - logkfi("[%s] failed with [%s] error: %d, try exit ...\n", mod->info.name, args, rc); + logkfi("[%s] init failed: %ld, try exit ...\n", mod->info.name, rc); (*mod->exit)(reserved); } free: + /* A failed load must undo the dependency references it took, exactly like + * the kernel module loader when init_module fails. */ + { + unsigned int i; + unsigned long flags = kp_private_spin_lock(&module_lock); + for (i = 0; i < mod->dep_count; i++) mod->deps[i]->export_refs--; + mod->dep_count = 0; + kp_private_spin_unlock(&module_lock, flags); + } if (mod->args) kvfree(mod->args); kp_free_exec(mod->start); free1: @@ -526,30 +731,46 @@ long load_module_ex(const void *data, int len, const char *args, const char *eve // todo: lock long unload_module(const char *name, void *__user reserved) { + struct module *mod; + unsigned long lock_flags; + unsigned int i; + long rc; + if (!name) return -EINVAL; + if (!kfunc(synchronize_rcu)) return -ENOSYS; logkfe("name: %s\n", name); rcu_read_lock(); - long rc = 0; - - struct module *mod = find_module(name); + lock_flags = kp_private_spin_lock(&module_lock); + mod = find_module(name); if (!mod) { - rc = -ENOENT; - goto out; - } - list_del(&mod->list); + kp_private_spin_unlock(&module_lock, lock_flags); + rcu_read_unlock(); + return -ENOENT; + } + if (mod->export_refs) { + logkfe("module %s is in use by %u other KPM(s)\n", name, mod->export_refs); + kp_private_spin_unlock(&module_lock, lock_flags); + rcu_read_unlock(); + return -EBUSY; + } + list_del_rcu(&mod->list); + kp_private_spin_unlock(&module_lock, lock_flags); + rcu_read_unlock(); + + kfunc(synchronize_rcu)(); rc = (*mod->exit)(reserved); + lock_flags = kp_private_spin_lock(&module_lock); + for (i = 0; i < mod->dep_count; i++) mod->deps[i]->export_refs--; + mod->dep_count = 0; + kp_private_spin_unlock(&module_lock, lock_flags); + if (mod->args) kvfree(mod->args); if (mod->ctl_args) kvfree(mod->ctl_args); - kp_free_exec(mod->start); kvfree(mod); - - logkfi("name: %s, rc: %d\n", name, rc); - -out: - rcu_read_unlock(); + logkfi("name: %s, rc: %ld\n", name, rc); return rc; } @@ -576,6 +797,11 @@ long load_module_path_event(const char *path, const char *args, const char *even goto out; } loff_t len = vfs_llseek(filp, 0, SEEK_END); + if (len <= 0 || len > KPM_LINK_MAX_IMAGE) { + rc = len < 0 ? len : -E2BIG; + set_kpm_load_result(reserved, rc, "module file size invalid or too large"); + goto close; + } logkfd("module size: %llx\n", len); vfs_llseek(filp, 0, SEEK_SET); @@ -619,7 +845,7 @@ long module_control0(const char *name, const char *ctl_args, char *__user out_ms int args_len = strlen(ctl_args); if (args_len <= 0) return -EINVAL; - logkfi("name %s, args: %s\n", name, ctl_args); + logkfi("control module: %s\n", name); long rc = 0; rcu_read_lock(); @@ -689,7 +915,7 @@ long notify_modules_event(const char *event, const char *args, void *__user rese rcu_read_lock(); struct module *pos; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { if (!pos->event || !*pos->event) continue; @@ -707,7 +933,7 @@ KP_EXPORT_SYMBOL(notify_modules_event); struct module *find_module(const char *name) { struct module *pos; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { if (!strcmp(name, pos->info.name)) { return pos; @@ -722,7 +948,7 @@ int get_module_nums() struct module *pos; int n = 0; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { n++; } @@ -741,7 +967,7 @@ int list_modules(char *out_names, int size) struct module *pos; int off = 0; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { off += snprintf(out_names + off, size - 1 - off, "%s\n", pos->info.name); } diff --git a/kernel/patch/module/relo.c b/kernel/patch/module/relo.c index 348dd7746..0fa0e6cbc 100644 --- a/kernel/patch/module/relo.c +++ b/kernel/patch/module/relo.c @@ -6,6 +6,7 @@ #include #include "insn.h" +#include "module.h" #define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX #define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16 @@ -40,7 +41,7 @@ static u64 do_reloc(enum aarch64_reloc_op reloc_op, void *place, u64 val) static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) { - u64 imm_mask = (1 << len) - 1; + u64 imm_mask = len == 64 ? ~0ULL : (1ULL << len) - 1; s64 sval = do_reloc(op, place, val); switch (len) { @@ -68,7 +69,7 @@ static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) * len bits (i.e the bottom len bits are not sign-extended and * the top bits are not all zero). */ - if ((u64)(sval + 1) > 2) return -ERANGE; + if (len != 64 && (u64)(sval + 1) >= 2) return -ERANGE; return 0; } @@ -162,7 +163,7 @@ static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val, int ls int apply_relocate(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, struct module *me) { - return 0; + return -ENOEXEC; }; int apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, @@ -312,8 +313,29 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, unsigned int sym break; case R_AARCH64_JUMP26: case R_AARCH64_CALL26: + if (val & 3) return -ENOEXEC; + if (!kpm_link_branch_in_range((unsigned long)loc, val)) { + val = kpm_link_plt(&me->link, me->start, val); + if (!val) return -ENOMEM; + } ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, AARCH64_INSN_IMM_26); break; + case KPM_RELOC_ADR_GOT_PAGE: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21, AARCH64_INSN_IMM_ADR); + break; + case KPM_RELOC_LD64_GOT_LO12_NC: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + overflow_check = false; + ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, AARCH64_INSN_IMM_12); + break; + case KPM_RELOC_GOT_LD_PREL19: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, AARCH64_INSN_IMM_19); + break; default: pr_err("unsupported RELA relocation: %llu\n", ELF64_R_TYPE(rel[i].r_info)); return -ENOEXEC; diff --git a/kpms/demo-exports/Makefile b/kpms/demo-exports/Makefile new file mode 100644 index 000000000..59cacd28a --- /dev/null +++ b/kpms/demo-exports/Makefile @@ -0,0 +1,29 @@ +ifndef TARGET_COMPILE + $(error TARGET_COMPILE not set) +endif + +ifndef KP_DIR + KP_DIR = ../.. +endif + +CC = $(TARGET_COMPILE)gcc +LD = $(TARGET_COMPILE)ld + +INCLUDE_DIRS := . include patch/include linux/include linux/arch/arm64/include linux/tools/arch/arm64/include + +INCLUDE_FLAGS := $(foreach dir,$(INCLUDE_DIRS),-I$(KP_DIR)/kernel/$(dir)) + +objs := exports.o + +all: exports.kpm + +exports.kpm: ${objs} + ${CC} -r -o $@ $^ + +%.o: %.c + ${CC} $(CFLAGS) $(INCLUDE_FLAGS) -Texports.lds -c -O2 -o $@ $< + +.PHONY: clean +clean: + rm -rf *.kpm + find . -name "*.o" | xargs rm -f diff --git a/kpms/demo-exports/exports.c b/kpms/demo-exports/exports.c new file mode 100644 index 000000000..d3422275a --- /dev/null +++ b/kpms/demo-exports/exports.c @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include +#include +#include +#include + +KPM_NAME("kpm-exports-demo"); +KPM_VERSION("1.0.0"); +KPM_LICENSE("GPL v2"); +KPM_AUTHOR("bmax121"); +KPM_DESCRIPTION("KernelPatch Module cross-KPM symbol export example"); + +/* KPM_EXPORT publishes this function to later KPMs, the same way an LKM + * publishes a symbol with EXPORT_SYMBOL. */ +int kpm_exports_add(int a, int b) +{ + return a + b; +} +KPM_EXPORT(kpm_exports_add); + +static long exports_init(const char *args, const char *event, void *__user reserved) +{ + pr_info("kpm exports init, event: %s, args: %s\n", event, args); + pr_info("kpm_exports_add(20, 22) = %d (importable by other KPMs)\n", kpm_exports_add(20, 22)); + return 0; +} + +static long exports_control0(const char *args, char *__user out_msg, int outlen) +{ + pr_info("kpm exports control0, args: %s\n", args); + return 0; +} + +static long exports_exit(void *__user reserved) +{ + pr_info("kpm exports exit\n"); + return 0; +} + +KPM_INIT(exports_init); +KPM_CTL0(exports_control0); +KPM_EXIT(exports_exit); diff --git a/kpms/demo-exports/exports.lds b/kpms/demo-exports/exports.lds new file mode 100644 index 000000000..22e36a21c --- /dev/null +++ b/kpms/demo-exports/exports.lds @@ -0,0 +1,5 @@ +SECTIONS { + .plt (NOLOAD) : { BYTE(0) } + .init.plt (NOLOAD) : { BYTE(0) } + .text.ftrace_trampoline (NOLOAD) : { BYTE(0) } +} diff --git a/kpms/demo-import/Makefile b/kpms/demo-import/Makefile new file mode 100644 index 000000000..33a91a30f --- /dev/null +++ b/kpms/demo-import/Makefile @@ -0,0 +1,29 @@ +ifndef TARGET_COMPILE + $(error TARGET_COMPILE not set) +endif + +ifndef KP_DIR + KP_DIR = ../.. +endif + +CC = $(TARGET_COMPILE)gcc +LD = $(TARGET_COMPILE)ld + +INCLUDE_DIRS := . include patch/include linux/include linux/arch/arm64/include linux/tools/arch/arm64/include + +INCLUDE_FLAGS := $(foreach dir,$(INCLUDE_DIRS),-I$(KP_DIR)/kernel/$(dir)) + +objs := import.o + +all: import.kpm + +import.kpm: ${objs} + ${CC} -r -o $@ $^ + +%.o: %.c + ${CC} $(CFLAGS) $(INCLUDE_FLAGS) -Timport.lds -c -O2 -o $@ $< + +.PHONY: clean +clean: + rm -rf *.kpm + find . -name "*.o" | xargs rm -f diff --git a/kpms/demo-import/import.c b/kpms/demo-import/import.c new file mode 100644 index 000000000..99ce085f1 --- /dev/null +++ b/kpms/demo-import/import.c @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include +#include +#include +#include + +KPM_NAME("kpm-import-demo"); +KPM_VERSION("1.0.0"); +KPM_LICENSE("GPL v2"); +KPM_AUTHOR("bmax121"); +KPM_DESCRIPTION("KernelPatch Module cross-KPM symbol import example"); + +/* Resolved at load time against the exports of already-loaded KPMs + * (kpm-exports-demo must be loaded first), exactly like an LKM importing + * another module's EXPORT_SYMBOL. */ +extern int kpm_exports_add(int a, int b); + +static long import_init(const char *args, const char *event, void *__user reserved) +{ + int v = kpm_exports_add(20, 22); + pr_info("kpm import init, event: %s\n", event); + pr_info("imported kpm_exports_add(20, 22) = %d\n", v); + return 0; +} + +static long import_control0(const char *args, char *__user out_msg, int outlen) +{ + pr_info("kpm import control0, args: %s\n", args); + return 0; +} + +static long import_exit(void *__user reserved) +{ + pr_info("kpm import exit\n"); + return 0; +} + +KPM_INIT(import_init); +KPM_CTL0(import_control0); +KPM_EXIT(import_exit); diff --git a/kpms/demo-import/import.lds b/kpms/demo-import/import.lds new file mode 100644 index 000000000..22e36a21c --- /dev/null +++ b/kpms/demo-import/import.lds @@ -0,0 +1,5 @@ +SECTIONS { + .plt (NOLOAD) : { BYTE(0) } + .init.plt (NOLOAD) : { BYTE(0) } + .text.ftrace_trampoline (NOLOAD) : { BYTE(0) } +} diff --git a/lkm/kpm/module.c b/lkm/kpm/module.c index 5abfd9ce0..8a6ceb044 100644 --- a/lkm/kpm/module.c +++ b/lkm/kpm/module.c @@ -11,6 +11,8 @@ */ #include "module.h" #include "relo.h" +#include +#include "../infra/symbol_resolver.h" #include "symbols.h" #include @@ -366,6 +368,96 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs, unsigned return true; } +extern struct kp_module modules; +static spinlock_t module_lock; + +/* LKM loader semantics: search the .kpm.export tables of already-loaded KPMs + * for @name and record the dependency edge on the importer. The runtime + * table wins first, like vmlinux exports shadow module exports. */ +static unsigned long kpm_module_export_lookup(const char *name, struct kp_module *importer) +{ + struct kp_module *pos; + unsigned long found = 0; + + if (!importer) return 0; + { + /* IRQ-safe: kp_kpm_safe_kallsyms_on_each_symbol() takes this same lock + * with spin_lock_irqsave() and can run from a KPM callback context. + * Mixing the two forms lets a same-CPU interrupt deadlock on it. */ + unsigned long flags; + spin_lock_irqsave(&module_lock, flags); + list_for_each_entry(pos, &modules.list, list) { + unsigned int i; + for (i = 0; i < pos->export_count; i++) { + if (!strcmp(name, pos->exports[i].name)) { + int edge = kpm_dep_record((void **)importer->deps, + &importer->dep_count, + KPM_DEP_MAX, pos); + if (edge < 0) { + /* Fail resolution: silently succeeding would allow the + * provider to be unloaded while consumer still uses it. */ + logke("dependency table full; cannot import %s from %s\n", + name, pos->info.name); + goto out; + } + /* Pin the provider before returning its address. The + * reference stays provisional through importer init and is + * rolled back if loading fails. */ + if (edge > 0) pos->export_refs++; + found = (unsigned long)pos->exports[i].target; + goto out; + } + } + } +out: + spin_unlock_irqrestore(&module_lock, flags); + } + return found; +} + +static unsigned long kpm_compat_lookup(const char *name, void *context) +{ + unsigned long addr = kp_kpm_symbol_lookup(name); + if (addr) return addr; + return kpm_module_export_lookup(name, context); +} + +static unsigned long kpm_function_lookup(const char *name, void *context) +{ + (void)context; + return (unsigned long)kp_resolve_symbol_variant(name); +} + +static unsigned long kpm_data_lookup(const char *name, void *context) +{ + (void)context; + return kallsyms_lookup_name(name); +} + +static unsigned long kpm_pointer_slot(unsigned long target, void *context) +{ + struct kp_module *mod = context; + return kpm_link_pointer(&mod->link, mod->start, target); +} + +/* Reserve pointer slots and a worst-case PLT/GOT entry per allocated RELA. */ +static int kpm_prepare_link(struct kp_module *mod, const struct kp_load_info *info) +{ + unsigned long count = info->sechdrs[info->index.sym].sh_size / sizeof(Elf_Sym); + unsigned int i; + if (count > KPM_LINK_MAX_IMAGE / KPM_LINK_SLOT_SIZE) return -E2BIG; + for (i = 1; i < info->hdr->e_shnum; i++) { + const Elf_Shdr *section = &info->sechdrs[i]; + if (section->sh_type != SHT_RELA || section->sh_info >= info->hdr->e_shnum || + !(info->sechdrs[section->sh_info].sh_flags & SHF_ALLOC)) continue; + if (section->sh_size % sizeof(Elf64_Rela)) return -ENOEXEC; + if (section->sh_size / sizeof(Elf64_Rela) > KPM_LINK_MAX_IMAGE / KPM_LINK_SLOT_SIZE - count) + return -E2BIG; + count += section->sh_size / sizeof(Elf64_Rela); + } + return kpm_link_reserve(&mod->link, &mod->size, count) ? -E2BIG : 0; +} + /* Change all symbols so that st_value encodes the pointer directly. */ static int simplify_symbols(struct kp_module *mod, struct kp_load_info *info) { @@ -378,32 +470,40 @@ static int simplify_symbols(struct kp_module *mod, struct kp_load_info *info) for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) { const char *name = info->strtab + sym[i].st_name; switch (sym[i].st_shndx) { - case SHN_COMMON: - if (!strncmp(name, "__gnu_lto", 9)) { - logkd("Please compile with -fno-common\n"); - ret = -ENOEXEC; - } - break; + case SHN_COMMON: + set_load_error(info, "COMMON symbol unsupported: compile with -fno-common and without LTO"); + ret = -ENOEXEC; + break; case SHN_ABS: break; - case SHN_UNDEF: - { - unsigned long addr = kp_kpm_symbol_lookup(name); - if (!addr) - addr = kallsyms_lookup_name(name); - if (!addr) { - logke("unknown symbol: %s\n", name); - if (!info->info.error_msg[0]) - snprintf(info->info.error_msg, sizeof(info->info.error_msg), - "unknown symbol: %s", name); - ret = -ENOENT; - break; - } - sym[i].st_value = addr; - break; - } - default: - secbase = info->sechdrs[sym[i].st_shndx].sh_addr; + case SHN_UNDEF: { + struct kpm_symbol_resolution resolved; + int rc = kpm_symbol_resolve(name, kpm_compat_lookup, kpm_function_lookup, + kpm_data_lookup, kpm_pointer_slot, mod, &resolved); + if (rc) { + if (rc == -1 && ELF_ST_BIND(sym[i].st_info) == STB_WEAK) { + sym[i].st_value = 0; + break; + } + logke("unresolved symbol: %s (%s)\n", name, + rc == -2 ? "link arena exhausted" : "unavailable on this kernel"); + if (!info->info.error_msg[0]) + snprintf(info->info.error_msg, sizeof(info->info.error_msg), + "unresolved symbol: %s (%s)", name, + rc == -2 ? "link arena exhausted" : "unavailable on this kernel"); + ret = rc == -2 ? -ENOMEM : -ENOENT; + break; + } + sym[i].st_value = resolved.address; + break; + } + default: + if (sym[i].st_shndx >= info->hdr->e_shnum) { + set_load_error(info, "invalid symbol section index"); + ret = -ENOEXEC; + break; + } + secbase = info->sechdrs[sym[i].st_shndx].sh_addr; sym[i].st_value += secbase; break; } @@ -470,8 +570,26 @@ static int rewrite_section_headers(struct kp_load_info *info) info->sechdrs[0].sh_addr = 0; for (int i = 1; i < info->hdr->e_shnum; i++) { Elf_Shdr *shdr = &info->sechdrs[i]; - if (shdr->sh_type != SHT_NOBITS && info->len < shdr->sh_offset + shdr->sh_size) + Elf_Shdr *strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (shdr->sh_name >= strings->sh_size || + !memchr(info->secstrings + shdr->sh_name, 0, strings->sh_size - shdr->sh_name)) + return -ENOEXEC; + if (shdr->sh_addralign && (shdr->sh_addralign & (shdr->sh_addralign - 1))) + return -ENOEXEC; + /* Subtraction, not sh_offset + sh_size: the sum can wrap for a + * crafted 64-bit offset and let the bounds check pass. */ + if (shdr->sh_type != SHT_NOBITS && + (shdr->sh_offset > info->len || shdr->sh_size > info->len - shdr->sh_offset)) return -ENOEXEC; + /* sh_name indexes the section string table; bound it before + * move_module() hands the name to strcmp(). */ + { + const Elf_Shdr *section_strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (shdr->sh_name >= section_strings->sh_size || + !memchr(info->secstrings + shdr->sh_name, 0, + section_strings->sh_size - shdr->sh_name)) + return -ENOEXEC; + } /* Mark all sections sh_addr with their address in the temporary image. */ shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset; } @@ -512,6 +630,11 @@ static int move_module(struct kp_module *mod, struct kp_load_info *info) mod->event = (mod_eventcall_t *)dest; if (!mod->info.base && !strcmp(".kpm.info", sname)) mod->info.base = (const char *)dest; + + if (!mod->exports && !strcmp(".kpm.export", sname)) { + mod->exports = (const struct kpm_export_entry *)dest; + mod->export_count = shdr->sh_size / sizeof(struct kpm_export_entry); + } } mod->info.name = info->info.name - info->info.base + mod->info.base; mod->info.version = info->info.version - info->info.base + mod->info.base; @@ -530,6 +653,16 @@ static int setup_load_info(struct kp_load_info *info) { int rc = 0; info->sechdrs = (void *)info->hdr + info->hdr->e_shoff; + if (!info->hdr->e_shstrndx || info->hdr->e_shstrndx >= info->hdr->e_shnum) { + set_load_error(info, "invalid section string table index"); + return -ENOEXEC; + } + Elf_Shdr *section_strings = &info->sechdrs[info->hdr->e_shstrndx]; + if (section_strings->sh_type != SHT_STRTAB || section_strings->sh_offset > info->len || + section_strings->sh_size > info->len - section_strings->sh_offset) { + set_load_error(info, "invalid section string table bounds"); + return -ENOEXEC; + } info->secstrings = (void *)info->hdr + info->sechdrs[info->hdr->e_shstrndx].sh_offset; if ((rc = rewrite_section_headers(info))) { @@ -580,6 +713,10 @@ static int setup_load_info(struct kp_load_info *info) if (info->sechdrs[i].sh_type == SHT_SYMTAB) { info->index.sym = i; info->index.str = info->sechdrs[i].sh_link; + if (info->index.str >= info->hdr->e_shnum) { + set_load_error(info, "invalid symbol string section index"); + return -ENOEXEC; + } info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset; break; } @@ -590,6 +727,31 @@ static int setup_load_info(struct kp_load_info *info) set_load_error(info, "module has no symbols (stripped?)"); return -ENOEXEC; } + Elf_Shdr *symbols = &info->sechdrs[info->index.sym]; + if (symbols->sh_size % sizeof(Elf_Sym) || symbols->sh_link >= info->hdr->e_shnum) { + set_load_error(info, "invalid ELF symbol table"); + return -ENOEXEC; + } + Elf_Shdr *strings = &info->sechdrs[symbols->sh_link]; + if (strings->sh_type != SHT_STRTAB || !strings->sh_size) { + set_load_error(info, "invalid ELF symbol strings"); + return -ENOEXEC; + } + Elf_Sym *table = (void *)info->hdr + symbols->sh_offset; + for (unsigned int sym_i = 0; sym_i < symbols->sh_size / sizeof(Elf_Sym); sym_i++) { + if (table[sym_i].st_name >= strings->sh_size || + !memchr(info->strtab + table[sym_i].st_name, 0, strings->sh_size - table[sym_i].st_name)) { + set_load_error(info, "unterminated ELF symbol name"); + return -ENOEXEC; + } + } + { + int export_sec = find_sec(info, ".kpm.export"); + if (export_sec && info->sechdrs[export_sec].sh_size % sizeof(struct kpm_export_entry)) { + set_load_error(info, "malformed .kpm.export section"); + return -ENOEXEC; + } + } return 0; } @@ -613,7 +775,6 @@ static int elf_header_check(struct kp_load_info *info) } struct kp_module modules = { 0 }; -static spinlock_t module_lock; /* Set while a KPM's init runs: during that window the module is not yet on * modules.list, but its image must already be shielded from the Qualcomm @@ -725,7 +886,7 @@ int kp_kpm_safe_kallsyms_on_each_symbol(kp_kallsyms_cb_t fn, void *data) static struct kp_module *kp_find_module(const char *name) { struct kp_module *pos; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { if (!strcmp(name, pos->info.name)) return pos; @@ -776,6 +937,10 @@ long kp_load_module(const void *data, int len, const char *args, const char *eve layout_sections(mod, info); layout_symtab(mod, info); + if ((rc = kpm_prepare_link(mod, info))) { + set_load_error(info, "module link arena too large or invalid"); + goto free; + } logkfe("KPM [%s] layout done size=0x%x\n", info->info.name, mod->size); if ((rc = move_module(mod, info))) { @@ -825,8 +990,11 @@ long kp_load_module(const void *data, int len, const char *args, const char *eve pr_emerg(KPLKM_TAG ": KPM [%s] init returned %ld\n", mod->info.name, rc); if (!rc) { - logkfi("[%s] succeed with [%s]\n", mod->info.name, args ? args : ""); - list_add_tail(&mod->list, &modules.list); + unsigned long lock_flags; + logkfi("[%s] succeed\n", mod->info.name); + spin_lock_irqsave(&module_lock, lock_flags); + list_add_tail_rcu(&mod->list, &modules.list); + spin_unlock_irqrestore(&module_lock, lock_flags); goto out; } else { set_load_error(info, "module init failed"); @@ -837,6 +1005,15 @@ long kp_load_module(const void *data, int len, const char *args, const char *eve } free: + /* A failed load must undo the dependency references it took. */ + { + unsigned int i; + unsigned long flags; + spin_lock_irqsave(&module_lock, flags); + for (i = 0; i < mod->dep_count; i++) mod->deps[i]->export_refs--; + mod->dep_count = 0; + spin_unlock_irqrestore(&module_lock, flags); + } if (mod->args) kvfree(mod->args); if (mod->start) @@ -850,40 +1027,50 @@ long kp_load_module(const void *data, int len, const char *args, const char *eve long kp_unload_module(const char *name, void __user *reserved) { - if (!name) - return -EINVAL; - logkfe("name: %s\n", name); - - rcu_read_lock(); - long rc = 0; - - struct kp_module *mod = kp_find_module(name); - if (!mod) { - rc = -ENOENT; - goto out; - } - list_del(&mod->list); - rc = kp_call_exit(mod->exit, reserved); - - if (mod->args) - kvfree(mod->args); - if (mod->ctl_args) - kvfree(mod->ctl_args); - - if (kp_set_memory_nx && mod->start) { - int npages = (mod->size + PAGE_SIZE - 1) >> PAGE_SHIFT; - kp_do_set_memory_nx((unsigned long)mod->start, npages); - } - - if (kp_module_memfree && mod->start) - kp_free_exec(mod->start); - kfree(mod); - - logkfi("name: %s, rc: %ld\n", name, rc); - -out: - rcu_read_unlock(); - return rc; + struct kp_module *mod; + unsigned long lock_flags; + unsigned int i; + long rc; + + if (!name) return -EINVAL; + logkfe("name: %s\n", name); + + rcu_read_lock(); + spin_lock_irqsave(&module_lock, lock_flags); + mod = kp_find_module(name); + if (!mod) { + spin_unlock_irqrestore(&module_lock, lock_flags); + rcu_read_unlock(); + return -ENOENT; + } + if (mod->export_refs) { + logkfe("module %s is in use by %u other KPM(s)\n", name, mod->export_refs); + spin_unlock_irqrestore(&module_lock, lock_flags); + rcu_read_unlock(); + return -EBUSY; + } + list_del_rcu(&mod->list); + spin_unlock_irqrestore(&module_lock, lock_flags); + rcu_read_unlock(); + + synchronize_rcu(); + rc = kp_call_exit(mod->exit, reserved); + + spin_lock_irqsave(&module_lock, lock_flags); + for (i = 0; i < mod->dep_count; i++) mod->deps[i]->export_refs--; + mod->dep_count = 0; + spin_unlock_irqrestore(&module_lock, lock_flags); + + if (mod->args) kvfree(mod->args); + if (mod->ctl_args) kvfree(mod->ctl_args); + if (kp_set_memory_nx && mod->start) { + int npages = (mod->size + PAGE_SIZE - 1) >> PAGE_SHIFT; + kp_do_set_memory_nx((unsigned long)mod->start, npages); + } + if (kp_module_memfree && mod->start) kp_free_exec(mod->start); + kfree(mod); + logkfi("name: %s, rc: %ld\n", name, rc); + return rc; } long kp_load_module_path(const char *path, const char *args, void __user *reserved) @@ -904,6 +1091,11 @@ long kp_load_module_path(const char *path, const char *args, void __user *reserv goto out; } loff_t len = vfs_llseek(filp, 0, SEEK_END); + if (len <= 0 || len > KPM_LINK_MAX_IMAGE) { + rc = len < 0 ? len : -E2BIG; + set_kpm_load_result(reserved, rc, "module file size invalid or too large"); + goto close; + } logkfd("module size: %llx\n", len); vfs_llseek(filp, 0, SEEK_SET); @@ -1015,7 +1207,7 @@ long kp_notify_modules_event(const char *event, const char *args, void __user *r rcu_read_lock(); struct kp_module *pos; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { if (!pos->event || !*pos->event) continue; @@ -1037,7 +1229,7 @@ int kp_get_module_nums(void) struct kp_module *pos; int n = 0; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { n++; } @@ -1057,7 +1249,7 @@ int kp_list_modules(char *out_names, int size) struct kp_module *pos; int off = 0; - list_for_each_entry(pos, &modules.list, list) + list_for_each_entry_rcu(pos, &modules.list, list) { off += snprintf(out_names + off, size - 1 - off, "%s\n", pos->info.name); } diff --git a/lkm/kpm/module.h b/lkm/kpm/module.h index 20aa5615e..cfae9d271 100644 --- a/lkm/kpm/module.h +++ b/lkm/kpm/module.h @@ -18,6 +18,8 @@ #include #include +#include +#include /* KPM ABI function types (from kpmodule.h): mod_initcall_t, mod_ctl0call_t, * mod_ctl1call_t, mod_exitcall_t, mod_eventcall_t. */ @@ -57,6 +59,14 @@ struct kp_module { unsigned int ro_size; void *start; + struct kpm_link_state link; + + /* LKM-style cross-KPM exports (KPM_EXPORT -> .kpm.export) + deps */ + const struct kpm_export_entry *exports; + unsigned int export_count; + struct kp_module *deps[KPM_DEP_MAX]; /* providers we import from */ + unsigned int dep_count; + unsigned int export_refs; /* modules importing from us */ struct list_head list; }; diff --git a/lkm/kpm/relo.c b/lkm/kpm/relo.c index e83de1c4a..9aef25be6 100644 --- a/lkm/kpm/relo.c +++ b/lkm/kpm/relo.c @@ -39,7 +39,7 @@ static u64 do_reloc(enum kp_aarch64_reloc_op reloc_op, void *place, u64 val) static int reloc_data(enum kp_aarch64_reloc_op op, void *place, u64 val, int len) { - u64 imm_mask = (1 << len) - 1; + u64 imm_mask = len == 64 ? ~0ULL : (1ULL << len) - 1; s64 sval = do_reloc(op, place, val); switch (len) { @@ -62,7 +62,7 @@ static int reloc_data(enum kp_aarch64_reloc_op op, void *place, u64 val, int len sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1); /* Overflow if the value is not representable in len bits. */ - if ((u64)(sval + 1) > 2) + if (len != 64 && (u64)(sval + 1) >= 2) return -ERANGE; return 0; @@ -135,7 +135,7 @@ static int reloc_insn_imm(enum kp_aarch64_reloc_op op, void *place, u64 val, int int kp_apply_relocate(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, struct kp_module *me) { - return 0; + return -ENOEXEC; } int kp_apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -281,10 +281,31 @@ int kp_apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, unsigned int case R_AARCH64_CONDBR19: ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, KP_INSN_IMM_19); break; - case R_AARCH64_JUMP26: - case R_AARCH64_CALL26: - ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, KP_INSN_IMM_26); - break; + case R_AARCH64_JUMP26: + case R_AARCH64_CALL26: + if (val & 3) return -ENOEXEC; + if (!kpm_link_branch_in_range((unsigned long)loc, val)) { + val = kpm_link_plt(&me->link, me->start, val); + if (!val) return -ENOMEM; + } + ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, KP_INSN_IMM_26); + break; + case KPM_RELOC_ADR_GOT_PAGE: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21, KP_INSN_IMM_ADR); + break; + case KPM_RELOC_LD64_GOT_LO12_NC: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + overflow_check = false; + ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, KP_INSN_IMM_12); + break; + case KPM_RELOC_GOT_LD_PREL19: + val = kpm_link_pointer(&me->link, me->start, val); + if (!val) return -ENOMEM; + ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, KP_INSN_IMM_19); + break; default: pr_err("unsupported RELA relocation: %llu\n", ELF64_R_TYPE(rel[i].r_info)); return -ENOEXEC; diff --git a/lkm/kpm/symbols.c b/lkm/kpm/symbols.c index 78f614a1c..f970d8527 100644 --- a/lkm/kpm/symbols.c +++ b/lkm/kpm/symbols.c @@ -1,7 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* Compatibility symbols exported by the in-kernel KP runtime to KPMs. */ +/* Compatibility symbols exported by the in-kernel KP runtime to KPMs. + * + * The table mirrors the kpimg .kp.symbol ABI (see checks/symbol_compat.py for + * the enforced cross-version matrix). Where the LKM has no backing mechanism + * (kstorage, SU allowlist, IKCONFIG, raw_syscalln) the symbol is NOT faked: + * those either stay absent or degrade explicitly, so an old KPM gets a precise + * "unavailable on this backend" load error instead of a dangling address. */ #include "symbols.h" +#include +#include + +#include "module.h" +#include "../infra/symbol_resolver.h" + #include #include #include @@ -146,6 +158,59 @@ struct kp_kpm_symbol { unsigned long addr; }; +/* ---- previous-generation ABI shims (see checks/symbol_compat.py) ---- + * + * kp_kconfig_*: the IKCONFIG blob is a kpimg preset (start_preset.kconfig_*). + * An insmod-loaded LKM has no preset and the kernel's own kernel_config_data + * is freed init memory, so report "unavailable" honestly: old KPMs branch on + * kp_kconfig_available() and degrade gracefully instead of failing to load. */ +static int kp_kpm_kconfig_available(void) +{ + return 0; +} + +static int kp_kpm_kconfig_enabled(const char *name) +{ + (void)name; + return 0; +} + +static const char *kp_kpm_kconfig_value(const char *name) +{ + (void)name; + return NULL; +} + +/* copy_to_user_stack: same contract as kpimg's utils.c. */ +static long kp_kpm_copy_to_user_stack(const void *data, int len) +{ + unsigned long sp = current_user_stack_pointer(); + if (!data || len <= 0) + return -EINVAL; + sp = (sp - (unsigned long)len) & ~0x7UL; + return copy_to_user((void __user *)sp, data, len) ? -EFAULT : (long)sp; +} + +/* get_random_u64 is not exported on GKI 5.10; resolve at first use, then fall + * back to the exported get_random_bytes. */ +static u64 kp_kpm_get_random_u64(void) +{ + static u64 (*fn)(void); + u64 v = 0; + if (!fn) + fn = (void *)kallsyms_lookup_name("get_random_u64"); + if (fn) + return fn(); + get_random_bytes(&v, sizeof(v)); + return v; +} + +/* Data-symbol ABI: KPMs declare `extern unsigned long *sys_call_table;` and + * dereference once, so the slot must hold the table base. arm64 GKI has no + * compat table; a NULL slot plus has_config_compat=0 is the stock contract. */ +static kp_syscall_fn_t *kp_kpm_sys_call_table_ptr; +static kp_syscall_fn_t *kp_kpm_compat_sys_call_table_ptr; + /* KPM headers declare printk/kallsyms_lookup_name/kallsyms_on_each_symbol as * function-pointer *variables* (extern void (*printk)(...)), so KPMs load the * pointer *stored at* the symbol address (adrp + ldr [x,#lo12] + blr). The @@ -348,6 +413,29 @@ static struct kp_kpm_symbol kp_kpm_symbols[] = { { "hook_uninstall", (unsigned long)hook_uninstall }, { "hook", (unsigned long)hook }, { "unhook", (unsigned long)unhook }, + + /* ---- previous-generation kpimg ABI (matrix: checks/symbol_compat.py) ---- + * The LKM only implements the table-rewrite hook strategy, so the inline + * and fp syscall-hook variants all route through the same wrappers. + * kp_kconfig_* degrade to "unavailable" (the IKCONFIG blob is a kpimg + * preset); sys_call_table slots expose the resolved table / NULL compat. */ + { "inline_wrap_syscalln", (unsigned long)kp_kpm_hook_syscalln }, + { "inline_unwrap_syscalln", (unsigned long)kp_kpm_unhook_syscalln }, + { "fp_wrap_syscalln", (unsigned long)kp_kpm_hook_syscalln }, + { "fp_unwrap_syscalln", (unsigned long)kp_kpm_unhook_syscalln }, + { "kp_kconfig_available", (unsigned long)kp_kpm_kconfig_available }, + { "kp_kconfig_enabled", (unsigned long)kp_kpm_kconfig_enabled }, + { "kp_kconfig_value", (unsigned long)kp_kpm_kconfig_value }, + { "copy_to_user_stack", (unsigned long)kp_kpm_copy_to_user_stack }, + { "get_random_u64", (unsigned long)kp_kpm_get_random_u64 }, + { "notify_modules_event", (unsigned long)kp_notify_modules_event }, + { "symbol_lookup_name", (unsigned long)kp_kpm_symbol_lookup }, + { "kallsyms_lookup_name_by_suffix", (unsigned long)kp_resolve_symbol_variant }, +/* The table symbols are typed as kp_syscall_fn_t pointers, matching the LKM + * syscall-table ABI. KPM data-symbol relocations still receive their + * address, then dereference it once. */ + { "sys_call_table", (unsigned long)&kp_kpm_sys_call_table_ptr }, + { "compat_sys_call_table", (unsigned long)&kp_kpm_compat_sys_call_table_ptr }, }; int kp_kpm_symbols_init(void) @@ -407,6 +495,11 @@ int kp_kpm_symbols_init(void) kp_kpm_has_config_compat = 0; + /* Previous-generation data-symbol ABI: sys_call_table mirrors the + * resolved table; arm64 GKI has no compat table. */ + kp_kpm_sys_call_table_ptr = kp_sys_call_table; + kp_kpm_compat_sys_call_table_ptr = NULL; + /* arm64 GKI: syscalls go through __arm64_sys_* wrappers, so the KPM must * parse syscall args with the wrapper ABI (narg + 1). */ kp_kpm_has_syscall_wrapper = 0;