/* * SLUB introspection LKM — exposes per-cache slab + sheaf topology to unpriv * via world-readable debugfs. Lets researchers SEE the layout on hardened * Linux 7.0+ kernels with the new sheaves architecture * (RANDOM_KMALLOC_CACHES + SLAB_FREELIST_RANDOM + INIT_ON_ALLOC + sheaves). * * Usage: insmod slab_observe.ko (root) * echo "kmalloc-rnd-04-256" > /sys/kernel/debug/slab_observe/target * cat /sys/kernel/debug/slab_observe/dump * * Research-only. Not for shipping. * * Linux 7.0 sheaves rearchitecture: per-CPU is no longer a linked freelist * over a single active slab page. It's an array of pre-batched object pointers * sourced from possibly-different slab pages via the per-NODE barn. We dump * both the partial-slab list AND the per-CPU sheaf array contents so callers * can see whether sequential allocs come from the same page or different pages. */ #include #include #include #include #include #include #include #include #include #include #include #include /* slab_mutex / slab_caches are not EXPORT_SYMBOL'd. Resolve via kallsyms * (which itself isn't exported on 5.7+; we re-fetch via a kprobe trick). */ typedef unsigned long (*kallsyms_lookup_name_fn)(const char *); static kallsyms_lookup_name_fn kln; static struct list_head *slab_caches_ptr; static struct mutex *slab_mutex_ptr; static int resolve_slab_internals(void) { /* kprobe trick: register a kprobe at "kallsyms_lookup_name" to get its addr. */ struct kprobe kp = { .symbol_name = "kallsyms_lookup_name" }; int rc = register_kprobe(&kp); if (rc < 0) { pr_err("slab_observe: register_kprobe failed: %d\n", rc); return rc; } kln = (kallsyms_lookup_name_fn)kp.addr; unregister_kprobe(&kp); if (!kln) { pr_err("slab_observe: kln NULL\n"); return -EFAULT; } slab_caches_ptr = (struct list_head *)kln("slab_caches"); slab_mutex_ptr = (struct mutex *)kln("slab_mutex"); if (!slab_caches_ptr || !slab_mutex_ptr) { pr_err("slab_observe: missing symbols caches=%px mutex=%px\n", slab_caches_ptr, slab_mutex_ptr); return -ENOENT; } pr_info("slab_observe: resolved slab_caches=%px slab_mutex=%px\n", slab_caches_ptr, slab_mutex_ptr); return 0; } /* ===== Internal SLUB structs (mm/slab.h, mm/slub.c — not exported) ===== * Mirror layout from /home/stc/exp/kernel-7.0.0-15/sauce-audit/linux-7.0.0 * mm/slab.h:74 (struct slab) and mm/slub.c:404,420,430. */ struct freelist_counters { void *freelist; union { unsigned long counters; struct { unsigned inuse:16; unsigned objects:15; unsigned frozen:1; unsigned int stride; }; }; }; struct mirror_slab { unsigned long flags; /* memdesc_flags_t */ struct kmem_cache *slab_cache; union { struct { struct list_head slab_list; struct freelist_counters fc; }; struct rcu_head rcu_head; }; unsigned int __page_type; atomic_t __page_refcount; #ifdef CONFIG_SLAB_OBJ_EXT unsigned long obj_exts; #endif }; struct mirror_slab_sheaf { union { struct rcu_head rcu_head; struct list_head barn_list; struct { unsigned int capacity; bool pfmemalloc; }; }; struct kmem_cache *cache; unsigned int size; int node; void *objects[]; }; struct mirror_slub_percpu_sheaves { /* local_trylock_t — same width as a long on most archs */ long lock; struct mirror_slab_sheaf *main; struct mirror_slab_sheaf *spare; struct mirror_slab_sheaf *rcu_free; }; struct mirror_node_barn { spinlock_t lock; struct list_head sheaves_full; struct list_head sheaves_empty; unsigned int nr_full; unsigned int nr_empty; }; struct mirror_kmem_cache_node { spinlock_t list_lock; unsigned long nr_partial; struct list_head partial; #ifdef CONFIG_SLUB_DEBUG atomic_long_t nr_slabs; atomic_long_t total_objects; struct list_head full; #endif struct mirror_node_barn *barn; }; /* Mirror of struct kmem_cache from mm/slab.h:197. Order/sizes must match * exactly through the `list` field. */ struct reciprocal_value_mirror { u32 m; u8 sh1, sh2; }; /* 6 bytes */ struct kmoo_mirror { unsigned int x; }; /* kmem_cache_order_objects */ struct mirror_kmem_cache { struct mirror_slub_percpu_sheaves __percpu *cpu_sheaves; unsigned long flags; unsigned long min_partial; unsigned int size; unsigned int object_size; struct reciprocal_value_mirror reciprocal_size; unsigned int offset; unsigned int sheaf_capacity; struct kmoo_mirror oo; struct kmoo_mirror min; unsigned int allocflags; /* gfp_t */ int refcount; void (*ctor)(void *); unsigned int inuse; unsigned int align; unsigned int red_left_pad; const char *name; struct list_head list; /* tail varies (kobj/random/random_seq/...) — don't touch */ }; /* Offset of `node` array in struct kmem_cache, derived from this kernel's * BTF (bpftool btf dump file /sys/kernel/btf/vmlinux). On resolute-desk * 7.0.0-15-generic the layout after `list` is: * list @ 96 (16) * kobj @ 112 (64) * random @ 176 (8) * remote_node_defrag_ratio @ 184 (4) + 4 pad * random_seq @ 192 (8) * useroffset @ 200 (4) * usersize @ 204 (4) * node[] @ 208 * If you port to a different kernel/config, re-derive via * bpftool btf dump file /sys/kernel/btf/vmlinux format raw | * grep -A 30 "STRUCT 'kmem_cache' " | grep node * and update this constant. */ #define KMEM_CACHE_NODE_OFFSET 208 #define OO_SHIFT 16 #define OO_MASK ((1U << OO_SHIFT) - 1) static inline struct mirror_kmem_cache_node * get_node0(struct mirror_kmem_cache *mh) { return *(struct mirror_kmem_cache_node **) ((char *)mh + KMEM_CACHE_NODE_OFFSET); } /* ============================================================ */ static char target_name[64] = "kmalloc-rnd-04-256"; /* Plain read/write for target — NO seq_file. */ static ssize_t target_read(struct file *f, char __user *buf, size_t count, loff_t *ppos) { size_t n = strnlen(target_name, sizeof(target_name)); char tmp[80]; int copied = scnprintf(tmp, sizeof(tmp), "%s\n", target_name); return simple_read_from_buffer(buf, count, ppos, tmp, copied); } static ssize_t target_write(struct file *f, const char __user *buf, size_t count, loff_t *ppos) { char tmp[64]; size_t n = count < sizeof(tmp) - 1 ? count : sizeof(tmp) - 1; if (copy_from_user(tmp, buf, n)) return -EFAULT; tmp[n] = 0; /* strip trailing whitespace */ while (n > 0 && (tmp[n-1] == '\n' || tmp[n-1] == ' ' || tmp[n-1] == '\r' || tmp[n-1] == '\t' || tmp[n-1] == 0)) tmp[--n] = 0; /* Empty writes are no-ops (don't clobber the existing target). */ if (n == 0) { *ppos += count; return count; } memcpy(target_name, tmp, n + 1); pr_info("slab_observe: target=[%s]\n", target_name); *ppos += count; return count; } static const struct file_operations target_fops = { .owner = THIS_MODULE, .read = target_read, .write = target_write, .llseek = noop_llseek, }; /* Find cache by name in slab_caches list. Walk via mirror struct since * struct kmem_cache is opaque to LKMs. The list field is at known offset * (after `name`). */ static struct mirror_kmem_cache *find_cache(const char *name) { struct mirror_kmem_cache *s, *found = NULL; mutex_lock(slab_mutex_ptr); list_for_each_entry(s, slab_caches_ptr, list) { if (s->name && strcmp(s->name, name) == 0) { found = s; break; } } mutex_unlock(slab_mutex_ptr); return found; } static int dump_show(struct seq_file *m, void *v) { struct mirror_kmem_cache *mh; int cpu; pr_info("slab_observe: dump_show entry, target=%s\n", target_name); mh = find_cache(target_name); pr_info("slab_observe: find_cache returned %px\n", mh); if (!mh) { seq_printf(m, "cache %s not found\n", target_name); return 0; } seq_printf(m, "cache=%s size=%u object_size=%u offset=%u sheaf_capacity=%u cpu_sheaves=%px\n", mh->name, mh->size, mh->object_size, mh->offset, mh->sheaf_capacity, mh->cpu_sheaves); pr_info("slab_observe: printed header, sheaf_cap=%u cpu_sheaves=%px\n", mh->sheaf_capacity, mh->cpu_sheaves); if (mh->sheaf_capacity == 0 || !mh->cpu_sheaves) { seq_printf(m, "(no per-CPU sheaves)\n"); pr_info("slab_observe: returning early (no sheaves)\n"); return 0; } pr_info("slab_observe: proceeding with sheaf dump\n"); /* Single per-CPU loop. Snapshot pointers via READ_ONCE, bounds-check before * walking objects[]. Diagnostic pr_info on each step. */ seq_printf(m, "\nper-CPU sheaves:\n"); for_each_online_cpu(cpu) { struct mirror_slub_percpu_sheaves *pcs; struct mirror_slab_sheaf *main_s, *spare_s, *rcu_s; pcs = per_cpu_ptr(mh->cpu_sheaves, cpu); pr_info("slab_observe: cpu=%d pcs=%px\n", cpu, pcs); if (!pcs) continue; main_s = READ_ONCE(pcs->main); spare_s = READ_ONCE(pcs->spare); rcu_s = READ_ONCE(pcs->rcu_free); pr_info("slab_observe: cpu=%d main=%px spare=%px rcu=%px\n", cpu, main_s, spare_s, rcu_s); seq_printf(m, " cpu=%d main=%px spare=%px rcu_free=%px\n", cpu, main_s, spare_s, rcu_s); if (main_s) { /* Note: main_s->capacity is in a union with rcu_head/barn_list and * is only meaningful for *prefilled* sheaves. For active main, we * cap iteration via the kmem_cache's sheaf_capacity. */ unsigned int sz = READ_ONCE(main_s->size); unsigned int cap_max = mh->sheaf_capacity; unsigned int i, n; pr_info("slab_observe: cpu=%d main size=%u (cap_max=%u)\n", cpu, sz, cap_max); if (sz > cap_max + 32) { /* sanity: shouldn't exceed capacity */ seq_printf(m, " main: stale size=%u cap_max=%u\n", sz, cap_max); continue; } n = sz > 64 ? 64 : sz; seq_printf(m, " main: size=%u objects[]:", sz); for (i = 0; i < n; i++) { void *o = READ_ONCE(main_s->objects[i]); seq_printf(m, " %px", o); } seq_printf(m, "\n"); } } pr_info("slab_observe: dump_show returning normally\n"); return 0; } static int dump_open(struct inode *i, struct file *f) { return single_open(f, dump_show, NULL); } static const struct file_operations dump_fops = { .owner = THIS_MODULE, .open = dump_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; /* probe: write an address, get back which slab page + slot index */ static unsigned long probe_addr; static int probe_show(struct seq_file *m, void *v) { struct mirror_kmem_cache *mh = find_cache(target_name); struct page *page; void *obj; if (!mh || !probe_addr) { seq_printf(m, "(set target and probe address first)\n"); return 0; } obj = (void *)probe_addr; page = virt_to_page(obj); if (!page) { seq_printf(m, "no page for %px\n", obj); return 0; } /* Convert page → slab via compound_head bit-0 trick (mm/slab.h:132). */ { unsigned long head = READ_ONCE(page->compound_head); struct page *spage = (head & 1) ? (struct page *)(head - 1) : page; struct mirror_slab *slab = (struct mirror_slab *)spage; unsigned long base = (unsigned long)page_address(spage); unsigned int slot = (unsigned long)((char *)obj - (char *)base) / mh->size; seq_printf(m, "addr=%px page=%px slab=%px slab_cache=%px slot=%u inuse=%d/%d\n", obj, page, slab, slab->slab_cache, slot, slab->fc.inuse, slab->fc.objects); seq_printf(m, " slab freelist=%px\n", slab->fc.freelist); } return 0; } static ssize_t probe_write(struct file *f, const char __user *buf, size_t len, loff_t *ppos) { char tmp[32]; unsigned long v; if (len >= sizeof(tmp)) return -EINVAL; if (copy_from_user(tmp, buf, len)) return -EFAULT; tmp[len] = 0; if (kstrtoul(tmp, 0, &v) < 0) return -EINVAL; probe_addr = v; return len; } static int probe_open(struct inode *i, struct file *f) { return single_open(f, probe_show, NULL); } static const struct file_operations probe_fops = { .owner = THIS_MODULE, .open = probe_open, .read = seq_read, .write = probe_write, .llseek = seq_lseek, .release = single_release, }; /* dump_all: scan ALL PFNs, find every slab page whose slab_cache == target. * This gives the COMPLETE inventory of slab pages owned by the cache (partial * + full + everything in-between), at the cost of an O(max_pfn) linear scan. * On a small VM (max_pfn ~ 2M), this runs in well under a second. * * Identification: PageSlab() / page->__page_type carries PG_slab tag. We * check via mirror_slab->slab_cache match (a partial false-positive on * non-slab pages is theoretically possible but the slab_cache pointer * range filter rules them out — kmem_cache is allocated from kmem_cache * itself, so we know the target's address). */ #include #include static int dump_all_show(struct seq_file *m, void *v) { struct mirror_kmem_cache *mh; unsigned long pfn, max_pfn_local; unsigned long matched = 0, scanned = 0; mh = find_cache(target_name); if (!mh) { seq_printf(m, "cache %s not found\n", target_name); return 0; } max_pfn_local = (unsigned long)kln("max_pfn"); if (max_pfn_local) { max_pfn_local = *(unsigned long *)max_pfn_local; } else { max_pfn_local = 4UL << 20; /* 4M PFN cap fallback ~16GB */ } seq_printf(m, "cache=%s scanning PFN [0, %lu)…\n", mh->name, max_pfn_local); for (pfn = 1; pfn < max_pfn_local; pfn++) { struct page *p; struct mirror_slab *s; if (!pfn_valid(pfn)) continue; p = pfn_to_page(pfn); if (!p) continue; scanned++; /* Quick filter: PageSlab via folio test — but keep it simple, just * check slab_cache match. The page's struct slab overlay puts * slab_cache at offset 8 (= mirror_slab->slab_cache). On non-slab * pages, that field is some other value (e.g. mapping for anon * pages) and won't equal mh. Equality is the filter. */ s = (struct mirror_slab *)p; if (s->slab_cache == (struct kmem_cache *)mh) { unsigned long addr = (unsigned long)page_address(p); seq_printf(m, "pfn=%lx addr=%lx inuse=%u/%u\n", pfn, addr, s->fc.inuse, s->fc.objects); matched++; } } seq_printf(m, "(scanned=%lu matched=%lu)\n", scanned, matched); return 0; } static int dump_all_open(struct inode *i, struct file *f) { return single_open(f, dump_all_show, NULL); } static const struct file_operations dump_all_fops = { .owner = THIS_MODULE, .open = dump_all_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; /* dump_slabs: walk node[0]->partial + node[0]->full lists, print each slab's * PFN/addr/inuse/objects/freelist. This is the persistent slab inventory — * pages survive cpu_sheaf cycles. The cross-cache empirical experiment compares * PFN sets across this file, NOT the per-CPU sheaf contents in `dump`. */ static void dump_slab_list(struct seq_file *m, struct list_head *head, const char *label, unsigned int objs_per_slab) { struct mirror_slab *slab; int n = 0; /* container_of from list_head -> mirror_slab via slab_list field. */ list_for_each_entry(slab, head, slab_list) { unsigned long addr; unsigned long pfn; unsigned int inuse, objects; if (n++ > 4096) { /* sanity cap */ seq_printf(m, " %s: (truncated at 4096)\n", label); break; } /* slab is layered on struct page; convert via virt_to_pfn(addr). */ addr = (unsigned long)page_address((struct page *)slab); pfn = page_to_pfn((struct page *)slab); /* fc.inuse and fc.objects are bitfields — caller holds list_lock so * plain reads are race-free. */ inuse = slab->fc.inuse; objects = slab->fc.objects; seq_printf(m, " %s: slab=%px addr=%lx pfn=%lx inuse=%u/%u freelist=%px\n", label, slab, addr, pfn, inuse, objects, slab->fc.freelist); } if (n == 0) seq_printf(m, " %s: (empty)\n", label); } static int dump_slabs_show(struct seq_file *m, void *v) { struct mirror_kmem_cache *mh; struct mirror_kmem_cache_node *n0; unsigned int objs_per_slab; unsigned long flags; mh = find_cache(target_name); if (!mh) { seq_printf(m, "cache %s not found\n", target_name); return 0; } objs_per_slab = mh->oo.x & OO_MASK; seq_printf(m, "cache=%s size=%u object_size=%u objs_per_slab=%u order=%u\n", mh->name, mh->size, mh->object_size, objs_per_slab, mh->oo.x >> OO_SHIFT); n0 = get_node0(mh); if (!n0) { seq_printf(m, "node[0] = NULL (cache offline?)\n"); return 0; } /* Hold list_lock across both partial and full to get a coherent snapshot. * The lock is irq-safe so use spin_lock_irqsave. */ spin_lock_irqsave(&n0->list_lock, flags); seq_printf(m, "node[0]: nr_partial=%lu nr_slabs=%ld total_objects=%ld\n", READ_ONCE(n0->nr_partial), atomic_long_read(&n0->nr_slabs), atomic_long_read(&n0->total_objects)); dump_slab_list(m, &n0->partial, "partial", objs_per_slab); #ifdef CONFIG_SLUB_DEBUG dump_slab_list(m, &n0->full, "full", objs_per_slab); #else seq_printf(m, " full: (CONFIG_SLUB_DEBUG off; full list not tracked)\n"); #endif spin_unlock_irqrestore(&n0->list_lock, flags); /* Barn (per-NODE pre-batched sheaves) — separate lock. */ if (n0->barn) { struct mirror_node_barn *barn = n0->barn; unsigned int nr_full, nr_empty; spin_lock_irqsave(&barn->lock, flags); nr_full = READ_ONCE(barn->nr_full); nr_empty = READ_ONCE(barn->nr_empty); spin_unlock_irqrestore(&barn->lock, flags); seq_printf(m, "node[0] barn: nr_full=%u nr_empty=%u\n", nr_full, nr_empty); } return 0; } static int dump_slabs_open(struct inode *i, struct file *f) { return single_open(f, dump_slabs_show, NULL); } static const struct file_operations dump_slabs_fops = { .owner = THIS_MODULE, .open = dump_slabs_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; static struct dentry *dbgdir; static int __init slab_observe_init(void) { int rc = resolve_slab_internals(); if (rc < 0) return rc; dbgdir = debugfs_create_dir("slab_observe", NULL); if (IS_ERR(dbgdir)) return PTR_ERR(dbgdir); debugfs_create_file("target", 0666, dbgdir, NULL, &target_fops); debugfs_create_file("dump", 0444, dbgdir, NULL, &dump_fops); debugfs_create_file("probe", 0666, dbgdir, NULL, &probe_fops); debugfs_create_file("dump_slabs", 0444, dbgdir, NULL, &dump_slabs_fops); debugfs_create_file("dump_all", 0444, dbgdir, NULL, &dump_all_fops); pr_info("slab_observe ready (sheaves-aware, node-walker, pfn-scan).\n"); return 0; } static void __exit slab_observe_exit(void) { debugfs_remove_recursive(dbgdir); } module_init(slab_observe_init); module_exit(slab_observe_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SLUB sheaves layout introspection");