utils.c

#include <stdbool.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include "breadcrumb_ring.h"
#include "child.h"
#include "debug.h"
#include "deferred-free.h"
#include "locks.h"
#include "objects.h"
#include "params.h"
#include "pc_format.h"
#include "pids.h"
#include "random.h"
#include "rnd.h"
#include "shm.h"
#include "signals.h"	// asb_copy_recover / asb_copy_active snapshot-copy guard
#include "stats.h"
#include "stats_ring.h"
#include "syscall.h"
#include "tables.h"
#include "trinity.h"
#include "utils.h"

/*
 * Use this allocator if you have an object a child writes to that you want
 * all other processes to see.
 *
 * Every allocation is tracked so that VM syscalls (munmap, madvise, mremap,
 * mprotect) can avoid clobbering trinity's own shared state.
 */

#ifdef CONFIG_GUARD_SHARED
/*
 * Runtime scope for the guard-page armour wired into __alloc_shared().
 * Initialised to GUARD_SCOPE_OFF; flipped to POOLS or ALL by parse_args()
 * when the operator passes --guard-shared[=pools|all].  Hot path:
 * __alloc_shared() reads this once per call.
 *
 *   OFF   - no guards, byte-identical to the legacy single-mmap path
 *           (modulo the runtime branch).  This is the production default.
 *   POOLS - guard only the long-lived regions tagged is_pool=true by
 *           their alloc site (kcov_shm, the shared str/obj heap, per-
 *           child childdata).  Bounded VMA cost, focused on the
 *           recurring corruption-witness clusters from the 2026-06-08
 *           overnight triages.
 *   ALL   - guard every alloc_shared() region, pool or not.  VMA cost
 *           scales with MAX_SHARED_ALLOCS; intended for short-run
 *           investigations where the writer might not be in the pools
 *           subset.  Warns + suggests raising vm.max_map_count at the
 *           flag-parse site.
 *
 * Off → __alloc_shared() and free_shared() collapse to today's exact
 * mmap / unregister behaviour at runtime as well, so a build that
 * compiled CONFIG_GUARD_SHARED in stays production-safe until the
 * operator opts in.
 */
enum guard_scope guard_shared_scope = GUARD_SCOPE_OFF;
#endif

static struct {
	unsigned long addr;
	unsigned long size;
#ifdef CONFIG_GUARD_SHARED
	/*
	 * 1 iff __alloc_shared() wrapped this region in PROT_NONE guard
	 * pages; 0 for a legacy single-mmap region (alloc_shared with
	 * guards off, or track_shared_region for an externally-mmap'd
	 * mapping).  free_shared() reads this to decide whether to
	 * derive a guarded span (PAGE + pages + PAGE) and munmap the
	 * whole span, or to munmap just (addr, size) like the legacy
	 * path.  child_fault_handler reads this in P1.2 to attribute a
	 * SEGV that lands in a guard page to its abutting region.
	 */
	uint8_t guarded;
#endif
} shared_regions[MAX_SHARED_ALLOCS];
unsigned int nr_shared_regions;

/*
 * Bounded overflow tail for registrations that arrive once
 * shared_regions[] is full.  Exists so range_overlaps_shared() (via the
 * bitmap, which is still updated) and range_in_tracked_shared() (via the
 * linear walk extension below) keep protecting fuzzed mm syscalls from
 * clobbering the untracked region instead of silently failing open.
 *
 * Intentionally small: 256 slots is "absorb a moderately over-budget
 * fleet host long enough to fail loudly and tell the operator to raise
 * MAX_SHARED_ALLOCS or move to dynamic resize", not "a second pool to
 * keep growing into".  Exhausting the tail BUG()s in both debug and
 * release; under-protection of a writable shared mapping is the failure
 * class the whole tracker exists to prevent and is never preferable to
 * a loud abort.
 */
#define SHARED_REGIONS_OVERFLOW_TAIL 256

static struct {
	unsigned long addr;
	unsigned long size;
#ifdef CONFIG_GUARD_SHARED
	uint8_t guarded;	/* parity with shared_regions[] above */
#endif
} shared_regions_overflow[SHARED_REGIONS_OVERFLOW_TAIL];
static unsigned int nr_shared_regions_overflow;

/*
 * Bitmap accelerator for range_overlaps_shared().  One bit per
 * SHARED_BITMAP_GRANULARITY-byte chunk of user VA; a set bit means at
 * least one byte in that chunk belongs to a registered shared region.
 *
 * The mm-syscall sanitisers (madvise/mremap/mprotect/munmap/mseal/mbind/
 * process_madvise/remap_file_pages/...) call range_overlaps_shared()
 * once per fuzzed call, often many times per child per second.  The
 * original linear scan over shared_regions[] is O(N) per query with N
 * easily reaching 100+ on a 32-child fleet (per-child childdata,
 * fd_event ring, kcov ring, plus the global reserve).  Replacing the
 * scan with this bitmap turns the hot path into one or two word loads
 * for the common single-page query.
 *
 * Granularity 2 MiB is the natural unit for the conservative
 * over-reject guarantee: any 2 MiB chunk that touches a shared region
 * gets its bit set, and a query whose footprint hits that chunk
 * rejects.  False positives are possible at chunk boundaries (a
 * non-shared page co-located in the same 2 MiB chunk as a shared
 * region rejects too), which is the SAFETY direction -- under-reject
 * would let a fuzzed mmap call clobber trinity's own shared state.
 *
 * Span 1<<47 covers the canonical x86_64 user VA on default
 * (4-level page table) kernels.  Regions registered outside the span
 * (5-level page tables, or unusually high mappings) are still recorded
 * in the authoritative shared_regions[]; shared_bitmap_mark() simply
 * no-ops for them and range_overlaps_shared() falls back to a linear
 * scan over shared_regions[] for above-span queries.  At 1 bit per
 * 2 MiB, the bitmap is
 * 1<<26 bits = 8 MiB of BSS, but it is mostly zero pages: only the
 * 4 KiB pages that cover actually-set bits ever fault in, so true
 * resident growth is in the kilobytes for a typical fleet host where
 * shared regions cluster in the mmap arena near 0x7f000000....
 */
#define SHARED_BITMAP_GRANULARITY_LOG2	21UL	/* 2 MiB per bit */
#define SHARED_BITMAP_VA_LOG2		47UL	/* 128 TiB user VA span */
#define SHARED_BITMAP_VA_SPAN		(1UL << SHARED_BITMAP_VA_LOG2)
#define SHARED_BITMAP_NBITS		(SHARED_BITMAP_VA_SPAN >> SHARED_BITMAP_GRANULARITY_LOG2)
#define SHARED_BITMAP_BITS_PER_WORD	(8UL * sizeof(unsigned long))
#define SHARED_BITMAP_NWORDS		(SHARED_BITMAP_NBITS / SHARED_BITMAP_BITS_PER_WORD)

static unsigned long shared_region_bitmap[SHARED_BITMAP_NWORDS];

/*
 * Per-chunk refcount paired with shared_region_bitmap above.  Multiple
 * tracked regions may live in the same 2 MiB chunk (every alloc_shared
 * call rounds up to a chunk for bitmap purposes; nothing forbids two
 * adjacent mmaps landing in the same chunk).  The bit must stay set
 * until the LAST tracked region in the chunk is removed -- clearing it
 * on the first untrack would flip the safety invariant from
 * "over-reject" to "under-reject" for the surviving region in the
 * chunk, exactly the failure mode this whole guard exists to prevent.
 *
 * uint16_t covers the worst-case occupancy by a comfortable margin
 * (MAX_SHARED_ALLOCS + overflow tail = 4352 << 65535) and bumps BSS
 * from 8 MiB (the bitmap alone) to 8 MiB + 128 MiB.  Same lazy-faulting
 * argument as the bitmap: only chunks touched by registrations ever
 * fault their backing page in, so true resident growth stays in the
 * tens of KiB for the typical clustered fleet-host layout.
 */
static uint16_t shared_region_refcount[SHARED_BITMAP_NBITS];

static inline bool shared_bitmap_test(unsigned long bit)
{
	return (shared_region_bitmap[bit / SHARED_BITMAP_BITS_PER_WORD] >>
		(bit % SHARED_BITMAP_BITS_PER_WORD)) & 1UL;
}

static inline void shared_bitmap_set(unsigned long bit)
{
	shared_region_bitmap[bit / SHARED_BITMAP_BITS_PER_WORD] |=
		1UL << (bit % SHARED_BITMAP_BITS_PER_WORD);
}

static inline void shared_bitmap_clear(unsigned long bit)
{
	shared_region_bitmap[bit / SHARED_BITMAP_BITS_PER_WORD] &=
		~(1UL << (bit % SHARED_BITMAP_BITS_PER_WORD));
}

/*
 * Mark every 2 MiB chunk that intersects [addr, addr+size).  Called
 * from the tail of alloc_shared() and track_shared_region() so the
 * bitmap stays in sync with shared_regions[].  size==0 is a no-op
 * (matches the "empty region overlaps nothing" semantics callers rely
 * on).  An out-of-span (or span-straddling) registration is a no-op
 * here: the bitmap only covers [0, SHARED_BITMAP_VA_SPAN), but
 * shared_regions[] is the authoritative registry and registration
 * already recorded the region there.  The query path falls back to a
 * linear scan over shared_regions[] for addresses the bitmap can't
 * cover, so the safety invariant ("no fuzzed mm syscall clobbers a
 * tracked region") still holds.
 */
static void shared_bitmap_mark(unsigned long addr, unsigned long size)
{
	unsigned long end, first, last, bit;

	if (size == 0)
		return;

	if (addr >= SHARED_BITMAP_VA_SPAN ||
	    size > SHARED_BITMAP_VA_SPAN - addr)
		return;

	end = addr + size - 1;
	first = addr >> SHARED_BITMAP_GRANULARITY_LOG2;
	last = end >> SHARED_BITMAP_GRANULARITY_LOG2;

	for (bit = first; bit <= last; bit++) {
		if (shared_region_refcount[bit] == UINT16_MAX) {
			outputerr("shared_bitmap_mark: refcount overflow at "
				  "chunk %lu for region 0x%lx+0x%lx\n",
				  bit, addr, size);
			BUG("shared region refcount overflow");
		}
		shared_region_refcount[bit]++;
		shared_bitmap_set(bit);
	}
}

/*
 * Inverse of shared_bitmap_mark().  Decrements the per-chunk refcount
 * for every 2 MiB chunk the range spans and clears the bitmap bit only
 * once the chunk's last tracked region is gone.  Called from
 * untrack_shared_region() after a matching shared_regions[] slot has
 * been located, so an inconsistency (refcount==0 on a chunk the caller
 * believes it tracked) is a tree-state bug worth BUG()ing on rather
 * than silently masking -- a stuck bit with refcount==0 would falsely
 * reject every fuzzed mm syscall touching the chunk forever.  An
 * out-of-span (or span-straddling) unmark is a no-op for the same
 * reason mark() no-ops above the span: the bitmap doesn't track those
 * addresses, so there is nothing to clear.
 */
static void shared_bitmap_unmark(unsigned long addr, unsigned long size)
{
	unsigned long end, first, last, bit;

	if (size == 0)
		return;

	if (addr >= SHARED_BITMAP_VA_SPAN ||
	    size > SHARED_BITMAP_VA_SPAN - addr)
		return;

	end = addr + size - 1;
	first = addr >> SHARED_BITMAP_GRANULARITY_LOG2;
	last = end >> SHARED_BITMAP_GRANULARITY_LOG2;

	for (bit = first; bit <= last; bit++) {
		if (shared_region_refcount[bit] == 0) {
			outputerr("shared_bitmap_unmark: refcount underflow at "
				  "chunk %lu for region 0x%lx+0x%lx\n",
				  bit, addr, size);
			BUG("shared region refcount underflow");
		}
		if (--shared_region_refcount[bit] == 0)
			shared_bitmap_clear(bit);
	}
}

/*
 * Size-bucket bitmap accelerator for range_overlaps_shared(): companion
 * to the address-keyed shared_region_bitmap above.  Bit i is set
 * whenever at least one tracked shared region currently falls into
 * size bucket i, where bucket i = floor(log2(len)) and covers regions
 * of len in [2^i, 2^(i+1)).  An empty bitmap (no tracked region of any
 * size) is the useful negative the address bitmap has to discover one
 * word at a time: one load here short-circuits the SHARED_BITMAP_NWORDS
 * word-scan over a multi-MiB query, plus the downstream byte-precise
 * walk that confirms a bitmap hit.
 *
 * Distinct concern from shared_region_bitmap above.  That bitmap
 * encodes WHERE tracked regions live (one bit per 2 MiB chunk of user
 * VA); this one encodes only WHETHER any tracked region exists in each
 * size class.  The two are wired in pairs: every register
 * (alloc_shared, track_shared_region, register_shared_overflow) calls
 * shared_bitmap_mark() AND tracked_size_mark(); every untrack (the
 * regular slot AND the overflow tail path in untrack_shared_region)
 * calls shared_bitmap_unmark() AND tracked_size_unmark().  Forgetting
 * the parallel call in a future refactor flips the size bitmap's
 * safety invariant from "empty ⇒ provably no regions" to "empty ⇒
 * silently under-reject"; shared_bitmap_self_check() asserts the
 * positive-path wiring at startup so that class of bug fails loudly.
 *
 * 64 buckets is the natural cap: a single unsigned long stores the
 * whole bitmap, and SHARED_BITMAP_VA_SPAN = 1<<47 bounds the largest
 * possible region at bucket 47 anyway -- buckets 48..63 stay zero on
 * any legitimate registration.  Per-bucket uint16_t refcount keeps the
 * bit set until the LAST region in that size class drops, mirroring
 * the shared_region_refcount discipline on the address bitmap; the
 * 4352-region worst case (MAX_SHARED_ALLOCS + SHARED_REGIONS_OVERFLOW_
 * TAIL) sits comfortably under UINT16_MAX, so a pathological run that
 * crowds every region into one bucket cannot overflow the counter.
 *
 * size==0 is a no-op for the same reason shared_bitmap_mark() no-ops
 * on size==0: the registering caller treats a zero-byte region as "no
 * region" and floor(log2(0)) is undefined, so suppressing the bump
 * here keeps the bitmaps in lockstep and avoids a spurious bucket-0
 * entry that no matching untrack would ever clear.
 */
#define TRACKED_SIZE_NBUCKETS	64
static unsigned long tracked_size_bm;
static uint16_t tracked_size_bucket_count[TRACKED_SIZE_NBUCKETS];

static inline unsigned int tracked_size_bucket(unsigned long len)
{
	return 63u - (unsigned int)__builtin_clzl(len);
}

static void tracked_size_mark(unsigned long len)
{
	unsigned int b;

	if (len == 0)
		return;

	b = tracked_size_bucket(len);
	if (b >= TRACKED_SIZE_NBUCKETS) {
		outputerr("tracked_size_mark: bucket %u out of range for len 0x%lx\n",
			  b, len);
		BUG("tracked_size bucket out of range");
	}
	if (tracked_size_bucket_count[b] == UINT16_MAX) {
		outputerr("tracked_size_mark: bucket %u refcount overflow for len 0x%lx\n",
			  b, len);
		BUG("tracked_size bucket refcount overflow");
	}
	if (tracked_size_bucket_count[b]++ == 0)
		tracked_size_bm |= 1UL << b;
}

static void tracked_size_unmark(unsigned long len)
{
	unsigned int b;

	if (len == 0)
		return;

	b = tracked_size_bucket(len);
	if (b >= TRACKED_SIZE_NBUCKETS) {
		outputerr("tracked_size_unmark: bucket %u out of range for len 0x%lx\n",
			  b, len);
		BUG("tracked_size bucket out of range");
	}
	if (tracked_size_bucket_count[b] == 0) {
		outputerr("tracked_size_unmark: bucket %u refcount underflow for len 0x%lx\n",
			  b, len);
		BUG("tracked_size bucket refcount underflow");
	}
	if (--tracked_size_bucket_count[b] == 0)
		tracked_size_bm &= ~(1UL << b);
}

/*
 * Handle a registration that arrived once shared_regions[] is full.
 *
 * The previous "warn once, then silently drop the region" policy turned
 * an over-budget host into the exact failure mode this whole tracker
 * exists to prevent: range_overlaps_shared() can no longer guard an
 * untracked writable MAP_SHARED region from a fuzzed
 * munmap/mremap/madvise/mprotect, so the next call that picks an
 * unlucky address scribbles trinity's own shared state and the
 * resulting crash looks like a kernel bug.  Silent under-protection of
 * a writable shared mapping is never preferable to a loud abort.
 *
 * New policy, per call:
 *
 *   - Always emit a LOUD outputerr() naming the caller PC (resolved via
 *     pc_to_string, same idiom as log_mprotect_failure()), the offending
 *     region, and the tail occupancy.  Per-call (not cap-once): the
 *     cap-once predecessor hid how badly the cap was over budget, which
 *     is the one piece of data needed to size a real fix.
 *
 *   - Under ASAN (the developer / debug build), BUG() immediately --
 *     overflow is a tree-state bug and we want a stack trace, not a
 *     production-shaped degradation.
 *
 *   - In release, register the region in the bounded overflow tail so
 *     the bitmap stays correct (shared_bitmap_mark already covers the
 *     range) and range_in_tracked_shared() can still match precisely.
 *     Bump shm->stats.shared_region_overflow so the over-budget state
 *     is visible in the periodic stats dump.
 *
 *   - If the overflow tail itself fills, BUG() in both debug and
 *     release.  Two layers of bounded storage is enough; a third would
 *     just be a slower path to the same silent-under-protection bug.
 */
static void register_shared_overflow(const char *who, unsigned long addr,
				     unsigned long size,
#ifdef CONFIG_GUARD_SHARED
				     bool guarded,
#endif
				     void *caller)
{
	char pcbuf[128];

	outputerr("shared_regions: %s overflow: region 0x%lx+0x%lx from %s; "
		  "MAX_SHARED_ALLOCS=%d exhausted, overflow tail at %u/%d -- "
		  "raise the cap or move shared_regions[] to dynamic resize\n",
		  who, addr, size,
		  pc_to_string(caller, pcbuf, sizeof(pcbuf)),
		  MAX_SHARED_ALLOCS,
		  nr_shared_regions_overflow, SHARED_REGIONS_OVERFLOW_TAIL);

#ifdef __SANITIZE_ADDRESS__
	BUG("shared_regions[] overflow (debug build)");
#else
	if (nr_shared_regions_overflow >= SHARED_REGIONS_OVERFLOW_TAIL) {
		outputerr("shared_regions: overflow tail also exhausted "
			  "(%d slots); refusing to leave region 0x%lx+0x%lx "
			  "untracked\n",
			  SHARED_REGIONS_OVERFLOW_TAIL, addr, size);
		BUG("shared_regions overflow tail exhausted");
	}

	shared_regions_overflow[nr_shared_regions_overflow].addr = addr;
	shared_regions_overflow[nr_shared_regions_overflow].size = size;
#ifdef CONFIG_GUARD_SHARED
	shared_regions_overflow[nr_shared_regions_overflow].guarded =
		guarded ? 1 : 0;
#endif
	shared_bitmap_mark(addr, size);
	tracked_size_mark(size);
	nr_shared_regions_overflow++;

	if (shm != NULL)
		__atomic_add_fetch(&shm->stats.shared_region_overflow, 1,
				   __ATOMIC_RELAXED);
#endif
}

#ifdef CONFIG_GUARD_SHARED
/*
 * Round len up to the nearest page boundary.  page_size is populated by
 * init_main_process() before parse_args() and any alloc_shared() caller,
 * so it is always non-zero by the time this is reachable.
 */
static size_t guard_pages_round_up(size_t len)
{
	size_t ps = (size_t)page_size;

	return (len + ps - 1) & ~(ps - 1);
}

/*
 * Recover (base, span) from the inner pointer + size of a guarded
 * region.  __alloc_shared() lays out a guarded mapping as
 *
 *   | leading guard (1 page) | unused fold | inner buffer | trailing guard (1 page) |
 *   ^base                     ^base+PAGE    ^ret           ^base+PAGE+pages
 *
 * with pages = round_up(size, page_size) and the inner buffer end-
 * aligned against the trailing guard so a forward overflow (buf[size])
 * traps at byte granularity.  Inverting:
 *
 *   pages = round_up(size, page_size)
 *   base  = ret - PAGE - (pages - size)
 *   span  = PAGE + pages + PAGE
 *
 * The size is stored in shared_regions[].size and the guarded bit is
 * stored alongside, so free_shared() needs no parallel side table to
 * unwind the layout.
 */
static void guard_pages_derive_span(void *ret, size_t size,
				    void **base_out, size_t *span_out)
{
	size_t ps = (size_t)page_size;
	size_t pages = guard_pages_round_up(size);
	char *base = (char *)ret - ps - (pages - size);

	*base_out = base;
	*span_out = ps + pages + ps;
}

/*
 * Mmap a guarded region: one VA span = leading-guard + usable-pages +
 * trailing-guard, with the inner buffer end-aligned against the
 * trailing guard.  Returns the inner pointer (the address callers see
 * and store in shared_regions[]), or MAP_FAILED on failure.  On
 * failure logs a single outputerr() line and leaves no VMA behind:
 * the leading-guard mprotect is reverted by munmap before return so
 * the caller can fall back to a non-guarded mmap without leaking VA.
 */
static void *guard_pages_alloc(size_t size)
{
	size_t ps = (size_t)page_size;
	size_t pages = guard_pages_round_up(size);
	size_t span = ps + pages + ps;
	char *base;

	base = mmap(NULL, span, PROT_READ | PROT_WRITE,
		    MAP_ANON | MAP_SHARED, -1, 0);
	if (base == MAP_FAILED) {
		outputerr("guard_pages_alloc: mmap %zu failure (span=%zu)\n",
			  size, span);
		return MAP_FAILED;
	}

	/* Drop the leading and trailing pages to PROT_NONE so any
	 * adjacent overflow traps in copy_*_user (kernel-side) or
	 * directly at the writer PC (userspace).  Splits the span into
	 * three VMAs (guard / usable / guard); the cost is +2 VMAs per
	 * guarded region.
	 *
	 * Both mprotects run once per guarded region at setup time
	 * (alloc_shared is called from init paths, not from the arg-gen
	 * hot loop), so the slow-path checker's blanket ban does not
	 * apply -- mark explicitly to keep the surface honest.
	 */
	/* check-static: slow-ok */
	if (mprotect(base, ps, PROT_NONE) != 0) {
		outputerr("guard_pages_alloc: mprotect(leading) failed: errno=%d\n",
			  errno);
		(void)munmap(base, span);
		return MAP_FAILED;
	}
	/* check-static: slow-ok */
	if (mprotect(base + ps + pages, ps, PROT_NONE) != 0) {
		outputerr("guard_pages_alloc: mprotect(trailing) failed: errno=%d\n",
			  errno);
		(void)munmap(base, span);
		return MAP_FAILED;
	}

	/* End-align the inner buffer against the trailing guard so a
	 * forward overflow at byte granularity (buf[size] = x) faults
	 * at the writer PC instead of corrupting the fold region. */
	return base + ps + (pages - size);
}

/*
 * Decide whether this allocation falls into the current guard scope.
 * GUARD_SCOPE_OFF gates everything off (legacy fast path).
 * GUARD_SCOPE_POOLS guards only is_pool=true alloc sites (kcov_shm,
 * shared str heap, childdata -- the long-lived regions the corruption
 * clusters keep pointing at).  GUARD_SCOPE_ALL guards every site.
 */
static bool guard_scope_covers(bool is_pool)
{
	switch (guard_shared_scope) {
	case GUARD_SCOPE_ALL:
		return true;
	case GUARD_SCOPE_POOLS:
		return is_pool;
	case GUARD_SCOPE_OFF:
	default:
		return false;
	}
}

/*
 * Classify a fault address against the guarded regions tracked in
 * shared_regions[].  Returns true and fills outs when @fault_addr
 * lands in either the leading or trailing PROT_NONE page abutting a
 * guarded region; false otherwise.
 *
 * Called from child_fault_handler() on every fatal-signal delivery
 * before the in-handler diagnostic path runs, so this MUST be async-
 * signal-safe: plain reads of file-scope arrays only -- no allocator,
 * no stdio, no lock, no libc call outside the POSIX 2024 sec 2.4.3 set.
 * shared_regions[] is published once at init time (single-threaded
 * parent context) and never mutated past first fork, so a child
 * handler observing it sees a stable snapshot.  The page_size global
 * is set in init_main_process(), also before any fork.
 *
 * @delta_out is the byte-distance from the fault address to the
 * nearest legitimate edge of the region: how far past the end for a
 * trailing-guard fault (fault_addr - region_end), or how far before
 * the start for a leading-guard fault (region_start - fault_addr - 1
 * mapped through 0).  Bounded by page_size by construction.
 */
bool guard_pages_classify(uintptr_t fault_addr,
			  uintptr_t *region_addr_out,
			  size_t *region_size_out,
			  bool *trailing_out,
			  unsigned long *delta_out)
{
	uintptr_t ps = (uintptr_t)page_size;
	uintptr_t leading_start, trailing_start;
	unsigned long pages;
	unsigned int i;

	if (ps == 0)
		return false;

	for (i = 0; i < nr_shared_regions; i++) {
		if (shared_regions[i].guarded == 0)
			continue;

		pages = (shared_regions[i].size + ps - 1) & ~(ps - 1);
		leading_start = shared_regions[i].addr - ps -
				(pages - shared_regions[i].size);
		trailing_start = shared_regions[i].addr +
				 shared_regions[i].size;
		/* trailing guard sits at base+PAGE+pages == addr+size +
		 * (pages - size); collapse via the layout invariant. */
		trailing_start = leading_start + ps + pages;

		if (fault_addr >= leading_start &&
		    fault_addr < leading_start + ps) {
			*region_addr_out = shared_regions[i].addr;
			*region_size_out = shared_regions[i].size;
			*trailing_out = false;
			*delta_out = (unsigned long)
				(shared_regions[i].addr - fault_addr);
			return true;
		}
		if (fault_addr >= trailing_start &&
		    fault_addr < trailing_start + ps) {
			*region_addr_out = shared_regions[i].addr;
			*region_size_out = shared_regions[i].size;
			*trailing_out = true;
			*delta_out = (unsigned long)
				(fault_addr -
				 (shared_regions[i].addr +
				  shared_regions[i].size));
			return true;
		}
	}

	for (i = 0; i < nr_shared_regions_overflow; i++) {
		if (shared_regions_overflow[i].guarded == 0)
			continue;

		pages = (shared_regions_overflow[i].size + ps - 1) & ~(ps - 1);
		leading_start = shared_regions_overflow[i].addr - ps -
				(pages - shared_regions_overflow[i].size);
		trailing_start = leading_start + ps + pages;

		if (fault_addr >= leading_start &&
		    fault_addr < leading_start + ps) {
			*region_addr_out = shared_regions_overflow[i].addr;
			*region_size_out = shared_regions_overflow[i].size;
			*trailing_out = false;
			*delta_out = (unsigned long)
				(shared_regions_overflow[i].addr - fault_addr);
			return true;
		}
		if (fault_addr >= trailing_start &&
		    fault_addr < trailing_start + ps) {
			*region_addr_out = shared_regions_overflow[i].addr;
			*region_size_out = shared_regions_overflow[i].size;
			*trailing_out = true;
			*delta_out = (unsigned long)
				(fault_addr -
				 (shared_regions_overflow[i].addr +
				  shared_regions_overflow[i].size));
			return true;
		}
	}

	return false;
}

/*
 * Name the current guard-shared scope for the startup banner.  Mirrors
 * the operator-facing spellings accepted by parse_args() so a log line
 * is grep-pasteable straight into a re-run command.  Stable string,
 * safe to call before parse_args (returns "off").
 */
const char *guard_shared_scope_name(void)
{
	switch (guard_shared_scope) {
	case GUARD_SCOPE_ALL:
		return "all";
	case GUARD_SCOPE_POOLS:
		return "pools";
	case GUARD_SCOPE_OFF:
	default:
		return "off";
	}
}

/*
 * Count shared_regions[] entries (plus the overflow tail) carrying a
 * set .guarded bit.  Matches the iteration in guard_pages_classify so
 * the banner's "(N regions guarded)" reads the same population the
 * fault-time classifier will see -- a zero here means no guard-page
 * VMA exists, even if the scope says POOLS / ALL.
 */
unsigned int guard_shared_count_guarded(void)
{
	unsigned int i, n = 0;

	for (i = 0; i < nr_shared_regions; i++)
		if (shared_regions[i].guarded)
			n++;
	for (i = 0; i < nr_shared_regions_overflow; i++)
		if (shared_regions_overflow[i].guarded)
			n++;
	return n;
}
#endif	/* CONFIG_GUARD_SHARED */

#ifdef CONFIG_GUARD_SHARED
/*
 * Primary shared-region allocator under CONFIG_GUARD_SHARED.  is_pool
 * tags long-lived regions (kcov_shm, shared str heap, childdata) so
 * --guard-shared=pools picks them up without dragging every per-child
 * tiny alloc into the VMA budget.  alloc_shared() below is the no-pool
 * entry point most call sites use; the three pool sites call
 * alloc_shared_pool() (a thin wrapper) which routes here with
 * is_pool=true.
 *
 * Behaviour matrix:
 *
 *   scope == OFF             -> single-mmap path (one runtime branch,
 *                               no extra syscalls).
 *   scope covers is_pool     -> guarded layout from guard_pages_alloc();
 *                               a guard-alloc failure logs and falls
 *                               back to the non-guarded path so the
 *                               run continues.
 *
 * Either path registers the INNER (ret, size) with shared_regions[] and
 * the bitmap; the guard pages are deliberately NOT tracked so the mm-
 * syscall sanitisers don't reject fuzzed calls against unrelated VA
 * that happens to share a 2 MiB bitmap chunk with a guard.  free_shared
 * inverts the layout via the guarded flag stored alongside.
 */
void * __alloc_shared(size_t size, bool is_pool)
{
	void *ret;
	bool guarded = false;

	if (guard_scope_covers(is_pool)) {
		ret = guard_pages_alloc(size);
		if (ret != MAP_FAILED)
			guarded = true;
	} else {
		ret = MAP_FAILED;
	}

	if (ret == MAP_FAILED) {
		ret = mmap(NULL, size, PROT_READ | PROT_WRITE,
			   MAP_ANON | MAP_SHARED, -1, 0);
	}
	if (ret == MAP_FAILED) {
		outputerr("mmap %zu failure\n", size);
		exit(EXIT_FAILURE);
	}
	/* poison with independently-random bytes to expose uninitialized reads. */
	{
		unsigned char *p = ret;
		size_t i;

		for (i = 0; i + sizeof(unsigned int) <= size; i += sizeof(unsigned int)) {
			unsigned int r = rnd_u32();
			memcpy(p + i, &r, sizeof(r));
		}
		for (; i < size; i++)
			p[i] = (unsigned char)rnd_u32();
	}

	if (nr_shared_regions < MAX_SHARED_ALLOCS) {
		shared_regions[nr_shared_regions].addr = (unsigned long) ret;
		shared_regions[nr_shared_regions].size = size;
		shared_regions[nr_shared_regions].guarded = guarded ? 1 : 0;
		shared_bitmap_mark((unsigned long) ret, size);
		tracked_size_mark(size);
		nr_shared_regions++;
	} else {
		register_shared_overflow("alloc_shared", (unsigned long) ret,
					 size, guarded,
					 __builtin_return_address(0));
	}

	return ret;
}

void * alloc_shared(size_t size)
{
	return __alloc_shared(size, false);
}

void * alloc_shared_pool(size_t size)
{
	return __alloc_shared(size, true);
}

/*
 * Inverse of __alloc_shared().  Removes the matching shared_regions[]
 * slot, then munmaps either the full guarded span (PAGE + pages + PAGE
 * derived from the stored size+guarded flag) or the legacy (ret, size)
 * range.  No current alloc_shared caller has a destructor -- all pool
 * regions live for the parent's lifetime -- but the symmetry is the
 * spec contract for free-path correctness, and a future caller that
 * needs to release a pool region (test harness, lifecycle rework) must
 * route through here so the guard VMAs are not leaked behind.  Misses
 * silently to match untrack_shared_region()'s tolerance for callers
 * whose alloc was a no-op (size==0) or whose addr+size pair never
 * matched a registered slot exactly.
 */
void free_shared(void *p, size_t size)
{
	void *base = p;
	size_t span = size;
	bool guarded = false;
	unsigned int i;

	if (p == NULL)
		return;

	for (i = 0; i < nr_shared_regions; i++) {
		if (shared_regions[i].addr != (unsigned long)p ||
		    shared_regions[i].size != size)
			continue;
		guarded = shared_regions[i].guarded != 0;
		break;
	}
	if (i == nr_shared_regions) {
		for (i = 0; i < nr_shared_regions_overflow; i++) {
			if (shared_regions_overflow[i].addr != (unsigned long)p ||
			    shared_regions_overflow[i].size != size)
				continue;
			guarded = shared_regions_overflow[i].guarded != 0;
			break;
		}
	}

	untrack_shared_region((unsigned long)p, size);

	if (guarded)
		guard_pages_derive_span(p, size, &base, &span);

	if (munmap(base, span) != 0)
		outputerr("free_shared: munmap(%p, %zu) failed: errno=%d\n",
			  base, span, errno);
}

#else	/* !CONFIG_GUARD_SHARED */

/*
 * Legacy single-mmap path.  Byte-identical to pre-guard-armor trinity.
 */
void * alloc_shared(size_t size)
{
	void *ret;

	ret = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);
	if (ret == MAP_FAILED) {
		outputerr("mmap %zu failure\n", size);
		exit(EXIT_FAILURE);
	}
	/* poison with independently-random bytes to expose uninitialized reads. */
	{
		unsigned char *p = ret;
		size_t i;

		for (i = 0; i + sizeof(unsigned int) <= size; i += sizeof(unsigned int)) {
			unsigned int r = rnd_u32();
			memcpy(p + i, &r, sizeof(r));
		}
		for (; i < size; i++)
			p[i] = (unsigned char)rnd_u32();
	}

	if (nr_shared_regions < MAX_SHARED_ALLOCS) {
		shared_regions[nr_shared_regions].addr = (unsigned long) ret;
		shared_regions[nr_shared_regions].size = size;
		shared_bitmap_mark((unsigned long) ret, size);
		tracked_size_mark(size);
		nr_shared_regions++;
	} else {
		register_shared_overflow("alloc_shared", (unsigned long) ret,
					 size, __builtin_return_address(0));
	}

	return ret;
}

#endif	/* CONFIG_GUARD_SHARED */

/*
 * Add an externally-mmap'd region to the shared_regions tracker so the
 * range_overlaps_shared() guards in the mm-syscall sanitisers refuse
 * fuzzed munmap/mremap/madvise/mprotect calls that target it.  Used by
 * code that mmaps via something other than alloc_shared() and still
 * needs the region protected from the fuzzer -- e.g., the per-child
 * kcov ring buffer mapped from /sys/kernel/debug/kcov.
 */
void track_shared_region(unsigned long addr, unsigned long size)
{
	if (nr_shared_regions < MAX_SHARED_ALLOCS) {
		shared_regions[nr_shared_regions].addr = addr;
		shared_regions[nr_shared_regions].size = size;
#ifdef CONFIG_GUARD_SHARED
		/* Externally-mmap'd, never guarded by __alloc_shared. */
		shared_regions[nr_shared_regions].guarded = 0;
#endif
		shared_bitmap_mark(addr, size);
		tracked_size_mark(size);
		nr_shared_regions++;
	} else {
		register_shared_overflow("track_shared_region", addr, size,
#ifdef CONFIG_GUARD_SHARED
					 false,
#endif
					 __builtin_return_address(0));
	}
}

/*
 * Inverse of track_shared_region() / alloc_shared() registration.
 * Removes the matching shared_regions[] entry (exact addr+size match)
 * and undoes the bitmap refcount/bit it contributed, so providers that
 * munmap their region on destructor (io_uring rings, kvm vCPU run
 * pages) stop accumulating stale slots and stop holding the bitmap bit
 * set after their VA has been recycled to something unrelated.
 *
 * Slot reuse uses swap-with-last compaction: the freed slot inherits
 * the array tail, nr_shared_regions decrements.  Nothing depends on
 * shared_regions[] order beyond shared_bitmap_self_check() peeking at
 * slot 0, and that runs once at init -- well before any destructor can
 * fire -- so the order disturbance is invisible to live code paths
 * (range_overlaps_shared and range_in_tracked_shared both walk the
 * whole array).
 *
 * Walks the overflow tail too: a provider whose registration was
 * parked there is no less tracked from the caller's perspective and
 * must be unregistered the same way; otherwise the tail would only
 * ever grow.
 *
 * A miss returns silently rather than BUG()ing: a caller may
 * legitimately untrack a region whose original track call was a no-op
 * (e.g. size==0), or whose addr+size pair doesn't exactly match a
 * registration (the slot allocator is exact-match only).  Silent miss
 * is the same shape as Linux's __ClearPageReserved on a non-Reserved
 * page -- the inverse of a "best effort" registration is best effort.
 */
void untrack_shared_region(unsigned long addr, unsigned long size)
{
	unsigned int i;

	for (i = 0; i < nr_shared_regions; i++) {
		if (shared_regions[i].addr != addr ||
		    shared_regions[i].size != size)
			continue;
		shared_bitmap_unmark(addr, size);
		tracked_size_unmark(size);
		shared_regions[i] = shared_regions[nr_shared_regions - 1];
		nr_shared_regions--;
		return;
	}

	for (i = 0; i < nr_shared_regions_overflow; i++) {
		if (shared_regions_overflow[i].addr != addr ||
		    shared_regions_overflow[i].size != size)
			continue;
		shared_bitmap_unmark(addr, size);
		tracked_size_unmark(size);
		shared_regions_overflow[i] =
			shared_regions_overflow[nr_shared_regions_overflow - 1];
		nr_shared_regions_overflow--;
		return;
	}
}

bool shared_size_mul(size_t a, size_t b, size_t *out)
{
	return !__builtin_mul_overflow(a, b, out);
}

/*
 * Size-bucketed freelist for shared heap recycling.
 *
 * Eight fixed-size buckets cover the common allocation sizes.  A freed slot
 * whose aligned size falls within a bucket is pushed onto that bucket's
 * lock-free stack; the next alloc of the same size pops it instead of
 * burning new bump space.  Allocations larger than 1024 bytes bypass the
 * freelist and use the bump allocator directly (documented below).
 *
 * The freelist link lives in the slot's own first sizeof(uintptr_t) bytes.
 * This is safe because the slot is not live when the link is written: the
 * caller has just handed it back to us, and we zero the rest of the slot
 * before writing the link so that a use-after-free still surfaces as zero-
 * byte reads rather than as a stale link pointer.
 *
 * CAS ordering: RELAXED is sufficient for the same reason as the bump
 * cursor — the caller publishes the resulting object via add_object()'s
 * RELEASE store, which is the actual synchronisation point for consumers.
 *
 * ABA safety via tagged pointer.  A naive lock-free stack with a single
 * pointer head is vulnerable to the classic ABA race: a popper reads
 * old_head=X and next=*X, but before its CAS another thread pops X, pops
 * X.next, and then pushes X back.  The CAS still sees head==X and succeeds,
 * but it installs the stale "next" value, leaving head pointing at a slot
 * that has already been handed back to a caller.  Two callers then think
 * they own the same slot; the resulting double-use corrupts whichever
 * obj struct was layered over the slot and faults later in unrelated
 * code paths far from the buggy free.
 *
 * The mitigation is a 16-bit version counter packed into the high bits of
 * the head word.  Each push and pop increments the version; the CAS
 * compares the full 64-bit (version, ptr) tuple.  The A→B→A sequence above
 * now leaves the head as (X, ver+2) rather than (X, ver), so the racer's
 * CAS fails on the version mismatch and it retries with a fresh load.
 *
 * The packing exploits the canonical-form invariant of x86_64 user-space
 * virtual addresses: only bits 0-47 are significant, and bit 47 is 0 for
 * any user-space pointer (kernel pointers have bit 47 == 1 and are
 * sign-extended into the upper 16 bits).  We therefore stash the version
 * counter in bits 48-63, recover the pointer with a 48-bit mask, and need
 * no sign extension on read.  The slot's stored "next" link is just the
 * raw pointer (no version bits) — the version lives only in the head.
 *
 * The 16-bit version is finite: a perfectly-timed sequence of exactly
 * 65536 push/pop pairs in the gap between a victim's load and CAS would
 * wrap the version back to its original value and re-expose the race.
 * For a process-bounded fuzzer with sub-microsecond critical sections
 * this is astronomically improbable; if it ever proves observable the
 * head can be widened to a 128-bit (ptr, version) tuple and switched to a
 * cmpxchg16b-based DWCAS without any caller change.
 */

/*
 * The packed (ptr, version) freelist head assumes the top 16 bits of every
 * freelist pointer are zero — i.e. a 48-bit canonical userspace VA range,
 * which is the x86-64 default and is not guaranteed on arm64 (52-bit
 * possible), s390x, riscv, or x86-64 with 5-level paging enabled.  Reject
 * the build explicitly so a future port hits the wall here instead of
 * shipping a subtly-broken allocator.  Removing this guard requires either
 * a DWCAS-based 128-bit head where available or a (struct ptr, generation)
 * variant guarded by a small lock — see the comment block above the head
 * declaration.
 */
#if !defined(__x86_64__)
#error "shm freelist (ptr, version) packing assumes 48-bit userspace VA — port requires DWCAS or struct+lock variant"
#endif

#define FREELIST_PTR_MASK	((uint64_t)((1ULL << 48) - 1))
#define FREELIST_VER_SHIFT	48

static const size_t bucket_sizes[NUM_SHM_FREELIST_BUCKETS] = {
	8, 16, 32, 64, 128, 256, 512, 1024
};

/*
 * Return the index of the smallest bucket that fits an allocation of
 * already-pointer-aligned size, or -1 if size exceeds all buckets.
 */
static int freelist_bucket(size_t aligned_size)
{
	unsigned int i;

	for (i = 0; i < NUM_SHM_FREELIST_BUCKETS; i++) {
		if (aligned_size <= bucket_sizes[i])
			return (int)i;
	}
	return -1;
}

/*
 * Pop a slot from the freelist bucket whose head lives at *head.
 * Returns NULL if the freelist is empty; otherwise returns a fully-zeroed
 * slot of slot_size bytes.  ABA-safe via the version tag in the high
 * bits of the head — see the block comment above.
 */
static void *freelist_pop(uint64_t *head, size_t slot_size)
{
	uint64_t old_tagged, new_tagged;
	uintptr_t ptr, next;
	uint16_t new_ver;

	old_tagged = __atomic_load_n(head, __ATOMIC_RELAXED);
	do {
		ptr = (uintptr_t)(old_tagged & FREELIST_PTR_MASK);
		if (ptr == 0)
			return NULL;
		/* The slot's first word holds the next pointer, with no
		 * version bits — versions live only in the head. */
		next = *(uintptr_t *)ptr;
		new_ver = (uint16_t)(old_tagged >> FREELIST_VER_SHIFT) + 1;
		new_tagged = ((uint64_t)next & FREELIST_PTR_MASK) |
			     ((uint64_t)new_ver << FREELIST_VER_SHIFT);
	} while (!__atomic_compare_exchange_n(head, &old_tagged, new_tagged,
					      false,
					      __ATOMIC_RELAXED,
					      __ATOMIC_RELAXED));

	memset((void *)ptr, 0, slot_size);
	return (void *)ptr;
}

/*
 * Push a slot onto the freelist bucket whose head lives at *head.
 * The entire slot (slot_size bytes) is zeroed first so that a use-after-
 * free reads as zero rather than stale data; then the freelist link is
 * written into the slot's first word before the CAS.  The version tag in
 * the head is incremented on every successful CAS to keep poppers safe
 * from ABA — see the block comment above.
 */
static void freelist_push(uint64_t *head, void *p, size_t slot_size)
{
	uint64_t old_tagged, new_tagged;
	uint16_t new_ver;

	memset(p, 0, slot_size);
	old_tagged = __atomic_load_n(head, __ATOMIC_RELAXED);
	do {
		/* Store only the pointer half of the previous head into our
		 * slot — the version stays in the head word. */
		*(uintptr_t *)p = (uintptr_t)(old_tagged & FREELIST_PTR_MASK);
		new_ver = (uint16_t)(old_tagged >> FREELIST_VER_SHIFT) + 1;
		new_tagged = ((uint64_t)(uintptr_t)p & FREELIST_PTR_MASK) |
			     ((uint64_t)new_ver << FREELIST_VER_SHIFT);
	} while (!__atomic_compare_exchange_n(head, &old_tagged, new_tagged,
					      false,
					      __ATOMIC_RELAXED,
					      __ATOMIC_RELAXED));
}

/*
 * Shared string heap — backing store for the string PAYLOADS of
 * string-shaped fields (filenames, label strings, fixed-size attr
 * buffers).  The owning obj struct itself comes from alloc_object()/
 * zmalloc_tracked and lives on the private per-process heap; only the
 * variable-length payload it points at is carved from this shared slab.
 *
 * The payload must be MAP_SHARED-before-fork: any obj struct reachable
 * from shm->global_objects[] must point only at memory that other
 * processes can also reach, and the only way to satisfy that for
 * payloads allocated after fork (the regen path) is to carve out of a
 * region that was already mapped before any child forked.
 *
 * Why a dedicated slab for payloads:
 *
 *   A single shared region sized only for variable-length string
 *   payloads keeps its failure mode independent and self-describing:
 *   an exhaustion message reads as "out of string slots" and nothing
 *   else.  Payloads are variable-length and dominated by short labels,
 *   so the pool can stay small.
 *
 * Capacity (1 MiB):
 *
 *   The string-bearing OBJ_GLOBAL providers (file/testfile, perf,
 *   memfd) hold short payloads — file paths typically under ~100 B,
 *   memfd labels under ~32 B, perf_event_attr buffers ~120 B (kernel
 *   caps the attr at PAGE_SIZE but trinity only memcpy's a struct's
 *   worth back into the obj for replay/dump).  At ~64 B average that
 *   is ~16k entries; at ~120 B per perf entry it is ~8.7k.  Slots up
 *   to 1024 B are recycled through the size-bucketed freelists on free,
 *   so steady-state occupancy tracks the live working set rather than
 *   total allocation volume.  Only above-bucket frees bump-and-leak;
 *   the 1 MiB ceiling is headroom for those.
 *
 * Why two entry points (alloc_shared_str + alloc_shared_strdup):
 *
 *   The eventual callers split cleanly into two shapes:
 *
 *     - strdup-style:  init_*_fds() and the .open regen hooks have
 *       a NUL-terminated source string (filename, label) and want a
 *       pointer to a stable shm copy of it.  alloc_shared_strdup()
 *       collapses the strlen+alloc+strcpy sequence at the call site.
 *
 *     - empty-buffer:  perf's open_perf_fd() has a fixed-size attr
 *       struct and just wants raw zeroed bytes to memcpy into.
 *       alloc_shared_str(sizeof(struct perf_event_attr)) hands back
 *       exactly that, with no NUL-termination contract.
 *
 *   Both share one heap and one free; the strdup variant is a thin
 *   wrapper around the primitive, not a separate allocator.
 *
 * Free strategy:
 *
 *   Poison the slot to zeros so a use-after-free surfaces as a "" /
 *   NUL-byte read rather than a live-looking string, then recycle via
 *   the size-bucketed freelist (see freelist_push/pop above).  Callers
 *   pass the original
 *   allocation size; for strdup-style strings that is strlen(p)+1 at
 *   free time, which is correct for a still-NUL-terminated string and
 *   harmless overshoot if it isn't (the buffer was zero-initialised).
 *   Slots too large for any freelist bucket are poisoned and leaked.
 */
/*
 * 1 MiB.  Slots up to 1024 B are returned to the size-bucketed
 * freelists on free, so steady-state occupancy tracks the live
 * working set under OBJ_LOCAL churn (post_*_fd callbacks plus
 * per-child fanout).  Above-bucket frees bump-and-leak — they lose a
 * slot apiece — and the 1 MiB ceiling sizes the headroom that path
 * needs over a sustained fuzz run.
 */
#define SHARED_STR_HEAP_SIZE (1U * 1024U * 1024U)

static char *shared_str_heap;
static size_t shared_str_heap_capacity;

static void shared_str_heap_init(void)
{
	/* Same pre-fork-mapping requirement as the obj heap: the first
	 * caller must run in the parent before any child forks, which
	 * holds for all current callers (init_*_fds via open_fds()
	 * before fork_children()).  Assert it so a future child-context
	 * caller cannot silently map a private heap behind the shared
	 * cursor, which would dangle string pointers across processes. */
	if (getpid() != mainpid) {
		outputerr("alloc_shared_str: heap init from child context "
			  "(pid %d, parent %d) -- would dangle shm pointers\n",
			  getpid(), mainpid);
		abort();
	}
	shared_str_heap_capacity = SHARED_STR_HEAP_SIZE;
	shared_str_heap = alloc_shared_pool(shared_str_heap_capacity);
}

void * alloc_shared_str(size_t size)
{
	size_t old_used, new_used;
	void *p;
	int bucket;

	if (size == 0)
		return NULL;

	if (shared_str_heap == NULL)
		shared_str_heap_init();

	/* Round up so each allocation starts pointer-aligned.  The
	 * primitive is generic — strings don't need it, but the empty-
	 * buffer callers (perf_event_attr) do, and one rule keeps the
	 * accounting simple. */
	size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);

	/* Try the freelist before touching the bump cursor. */
	bucket = freelist_bucket(size);
	if (bucket >= 0) {
		p = freelist_pop(&shm->shared_str_freelist[bucket],
				 bucket_sizes[bucket]);
		if (p != NULL)
			return p;
		/* Round bump to bucket size so the first free of a bump
		 * slot doesn't overrun the next slot via freelist_push's
		 * bucket-size memset. */
		size = bucket_sizes[bucket];
	}

	/* Lock-free bump via CAS on the shm-resident cursor.  RELAXED
	 * is sufficient because the caller publishes the obj (and therefore the string pointer)
	 * to consumers via add_object()'s RELEASE store on
	 * num_entries. */
	old_used = __atomic_load_n(&shm->shared_str_heap_used,
				   __ATOMIC_RELAXED);
	do {
		new_used = old_used + size;
		if (new_used > shared_str_heap_capacity) {
			outputerr("alloc_shared_str: heap exhausted "
				  "(cap %zu, used %zu, req %zu)\n",
				  shared_str_heap_capacity, old_used,
				  size);
			return NULL;
		}
	} while (!__atomic_compare_exchange_n(&shm->shared_str_heap_used,
					      &old_used, new_used,
					      false,
					      __ATOMIC_RELAXED,
					      __ATOMIC_RELAXED));

	p = shared_str_heap + old_used;
	memset(p, 0, size);
	return p;
}

char * alloc_shared_strdup(const char *src)
{
	size_t len;
	char *dst;

	if (src == NULL)
		return NULL;

	len = strlen(src) + 1;
	dst = alloc_shared_str(len);
	if (dst == NULL)
		return NULL;

	memcpy(dst, src, len);
	return dst;
}

void free_shared_str(void *p, size_t size)
{
	int bucket;

	if (p == NULL || size == 0)
		return;

	size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);

	bucket = freelist_bucket(size);
	if (bucket >= 0) {
		freelist_push(&shm->shared_str_freelist[bucket], p,
			      bucket_sizes[bucket]);
		return;
	}

	/* Size above all buckets — poison and leak (bump-and-leak fallback). */
	memset(p, 0, size);
}

/*
 * Render a PROT_* mask as "READ|WRITE|EXEC" into the caller-provided
 * buffer.  Empties to "NONE" for prot==PROT_NONE so the diagnostic line
 * is never silently truncated to nothing between the brackets.  Unknown
 * upper bits (PROT_GROWSDOWN, pkey bits, ...) are left to the raw
 * 0x%x rendering at the call site.
 */
static void prot_to_string(int prot, char *buf, size_t buflen)
{
	int n = 0;
	int written;

	if (buf == NULL || buflen == 0)
		return;

	buf[0] = '\0';

	/* snprintf returns the would-have-written length even when truncated.
	 * Naive `n += snprintf(buf+n, buflen-n, ...)` advances n past buflen
	 * once the buffer fills, so the next call writes outside buf.  Currently
	 * safe at buflen=32 with the three short strings but identical cliff
	 * to the stats.c stack-depth histogram cumulator.  Bound n each step. */
	if ((prot & PROT_READ) && (size_t)n < buflen) {
		written = snprintf(buf + n, buflen - (size_t)n, "READ");
		if (written > 0)
			n += written;
	}
	if ((prot & PROT_WRITE) && (size_t)n < buflen) {
		written = snprintf(buf + n, buflen - (size_t)n,
				   "%sWRITE", n ? "|" : "");
		if (written > 0)
			n += written;
	}
	if ((prot & PROT_EXEC) && (size_t)n < buflen) {
		written = snprintf(buf + n, buflen - (size_t)n,
				   "%sEXEC", n ? "|" : "");
		if (written > 0)
			n += written;
	}

	if (n == 0)
		snprintf(buf, buflen, "NONE");
}

static const char *mprotect_errstr(int err)
{
	switch (err) {
	case ENOMEM:	return "ENOMEM";
	case EACCES:	return "EACCES";
	case EINVAL:	return "EINVAL";
	case EAGAIN:	return "EAGAIN";
	case EFAULT:	return "EFAULT";
	default:	return "unknown error";
	}
}

void log_mprotect_failure(void *addr, size_t len, int prot,
			  void *caller, int err)
{
	char protbuf[32];
	char pcbuf[128];

	prot_to_string(prot, protbuf, sizeof(protbuf));
	outputerr("mprotect(addr=%p, len=%zu, prot=0x%x [%s]) failed at %s: %s\n",
		  addr, len, prot, protbuf,
		  pc_to_string(caller, pcbuf, sizeof(pcbuf)),
		  mprotect_errstr(err));
}

/*
 * Self-check: confirm range_overlaps_shared()'s bitmap accelerator
 * actually rejects the first registered region.  Catches construction
 * regressions (a future refactor that forgets to call
 * shared_bitmap_mark() at registration would otherwise fail open and
 * silently let the fuzzer clobber trinity's own shared state).  Runs
 * once -- the bitmap only grows with new registrations, so a single
 * positive assert is sufficient to prove the wiring works. */
void shared_bitmap_self_check(void)
{
	static bool checked;
	unsigned long base, bit;

	if (checked || nr_shared_regions == 0)
		return;
	checked = true;

	base = shared_regions[0].addr;
	/*
	 * The bitmap only tracks addresses inside its span; above-span
	 * registrations are recorded in shared_regions[] alone and the
	 * query path falls back to a linear scan for them.  Asserting on
	 * such an entry would read past the bitmap.
	 */
	if (base < SHARED_BITMAP_VA_SPAN) {
		bit = base >> SHARED_BITMAP_GRANULARITY_LOG2;
		if (!shared_bitmap_test(bit)) {
			outputerr("range_overlaps_shared bitmap missing first region "
				  "@ 0x%lx (bit %lu)\n", base, bit);
			BUG("shared region bitmap inconsistent");
		}
	}

	/*
	 * Companion size-bucket bitmap should also reflect the first
	 * registered region: any region with non-zero size lands in some
	 * bucket, so tracked_size_bm cannot be empty here.  Catches a
	 * future refactor that wires shared_bitmap_mark() but forgets the
	 * parallel tracked_size_mark() call -- silent under-protection of
	 * the size short-circuit (always-true skip on an empty bitmap)
	 * would defeat the bypass counter on every call.
	 */
	if (shared_regions[0].size != 0 && tracked_size_bm == 0) {
		outputerr("tracked_size_bm empty despite first region size 0x%lx\n",
			  shared_regions[0].size);
		BUG("tracked_size bitmap inconsistent");
	}
}

/* Tunable: how often range_overlaps_shared() emits a -v summary line.
 * Lower = noisier, higher = blunter. */
#define RANGE_OVERLAPS_SHARED_REJECT_REPORT_INTERVAL 10000

/* Sibling tunable for the size-bucket bitmap short-circuit path.
 * Same cadence as the rejects line above so the two -v summaries
 * stay in lockstep when both fire on the same child. */
#define RANGE_OVERLAPS_SHARED_BM_SKIP_REPORT_INTERVAL 10000

/*
 * Exact byte-range overlap confirmation after a bitmap hit.  The
 * bitmap rounds tracked regions up to 2 MiB chunks, so a hit means
 * "some tracked region lives in a chunk that touches the query" --
 * not that the query and the region share a byte.  Walking the two
 * region arrays here resolves the false positives in which a valid,
 * disjoint mm-syscall range shares a 2 MiB chunk with a tracked
 * allocation; the bitmap previously rejected those unconditionally,
 * losing real munmap / mremap / madvise / mprotect / mseal / mbind
 * coverage on hosts where allocations cluster.
 *
 * Semantics match the pre-bitmap byte-precise test:
 *   - a non-empty range [addr, end) overlaps [rstart, rend) iff
 *     addr < rend && rstart < end (standard interval overlap);
 *   - an empty (len == 0) range matches iff @addr is strictly inside
 *     a region (rstart <= addr < rend), which is the only empty-range
 *     case the original test accepted.
 *
 * Zero-size tracked regions never overlap anything, matching the
 * alloc_shared() / track_shared_region() callers that treat size==0
 * as "no region" (and matching shared_bitmap_mark()'s no-op on size 0).
 *
 * Returning false here only removes false-positive rejects; it never
 * accepts a query that truly overlaps a tracked region, so the safety
 * invariant the callers rely on (no fuzzed mm-syscall clobbers a
 * trinity-owned shared mapping) is preserved.
 */
static bool shared_regions_exact_overlap(unsigned long addr,
					 unsigned long len,
					 unsigned long end)
{
	unsigned int i;

	for (i = 0; i < nr_shared_regions; i++) {
		unsigned long rstart = shared_regions[i].addr;
		unsigned long rsize = shared_regions[i].size;
		unsigned long rend;

		if (rsize == 0)
			continue;
		rend = rstart + rsize;

		if (len == 0) {
			if (addr >= rstart && addr < rend)
				return true;
		} else {
			if (addr < rend && rstart < end)
				return true;
		}
	}

	for (i = 0; i < nr_shared_regions_overflow; i++) {
		unsigned long rstart = shared_regions_overflow[i].addr;
		unsigned long rsize = shared_regions_overflow[i].size;
		unsigned long rend;

		if (rsize == 0)
			continue;
		rend = rstart + rsize;

		if (len == 0) {
			if (addr >= rstart && addr < rend)
				return true;
		} else {
			if (addr < rend && rstart < end)
				return true;
		}
	}

	return false;
}

/* Last syscall to trip a range_overlaps_shared() reject.  Last-write-wins;
 * a coarse hint for which sanitiser is doing the most work, not a precise
 * audit trail.  Process-local statics: each child has its own copy, the
 * writer and the reader (the periodic -v summary below) live in the same
 * single-threaded child, so plain accesses suffice.  Torn reads are
 * acceptable for a diagnostic anyway. */
static unsigned int last_reject_syscall_nr;
static unsigned char last_reject_do32bit;
static unsigned char last_reject_have_syscall;

bool range_overlaps_shared(unsigned long addr, unsigned long len)
{
	unsigned long end, check_end, first, last;
	bool overlap = false;
	unsigned long n;
	struct childdata *child;

	/* Treat wrapped ranges as overlapping so callers reject them. */
	if (len != 0 && addr > ULONG_MAX - len)
		return true;

	end = addr + len;

	/*
	 * Size-bucket bitmap short-circuit: when no tracked region of any
	 * size class exists, the address-keyed shared_region_bitmap is
	 * also empty by construction, and the word-scan below would walk
	 * SHARED_BITMAP_NWORDS words to confirm.  One load on
	 * tracked_size_bm proves the same negative in O(1).  This is the
	 * useful empty-fleet / pre-registration / fully-untracked state;
	 * it also covers the bypass condition from the spec ("len exceeds
	 * every set bit's bucket bound") vacuously when no bits are set.
	 *
	 * Per-process rate-limited counter feeds the -v summary below.
	 * Each child has its own static via fork CoW, mirroring the
	 * cadence of the rejects line at the tail of this function.
	 */
	if (tracked_size_bm == 0) {
		static unsigned long local_skip;
		unsigned long s;

		s = ++local_skip;
		if (verbosity > 1 &&
		    (s % RANGE_OVERLAPS_SHARED_BM_SKIP_REPORT_INTERVAL) == 0)
			output(1, "range_overlaps_shared_bm_skip: %lu cumulative bitmap-empty short-circuits\n",
				s);
		return false;
	}

	/* Bitmap accelerator used as a fast NEGATIVE prefilter:
	 * O(ceil(len/2MB)+1) bit reads to rule out the common case where
	 * no tracked region lives in any 2 MiB chunk the query touches.
	 * The bitmap only covers [0, SHARED_BITMAP_VA_SPAN); queries that
	 * start above the span (or straddle it) cannot use the bitmap as
	 * an authoritative negative -- shared_bitmap_mark() no-ops for
	 * those addresses, and the authoritative shared_regions[] linear
	 * scan below answers the membership question for them.  A
	 * zero-length probe collapses to a single bit read on the chunk
	 * containing addr.
	 *
	 * A bitmap HIT is only a candidate: it means at least one
	 * tracked region lives in a chunk that touches the query, but
	 * chunk rounding turns disjoint ranges in the same chunk into
	 * false positives.  The exact byte-range walk below confirms the
	 * hit (or clears it) before we reject. */
	if (addr < SHARED_BITMAP_VA_SPAN) {
		check_end = end;
		if (check_end > SHARED_BITMAP_VA_SPAN)
			check_end = SHARED_BITMAP_VA_SPAN;

		first = addr >> SHARED_BITMAP_GRANULARITY_LOG2;
		if (check_end > addr)
			last = (check_end - 1) >> SHARED_BITMAP_GRANULARITY_LOG2;
		else
			last = first;

		/* Word-at-a-time scan over shared_region_bitmap[]:
		 * mask the partial first/last words, then sweep any
		 * middle words and skip the zero-word common case in a
		 * single load.  A mostly-empty bitmap with a multi-MiB
		 * query length now costs one load per 64 chunks (128 MiB
		 * of VA), not one per chunk. */
		{
			unsigned long first_word = first / SHARED_BITMAP_BITS_PER_WORD;
			unsigned long last_word  = last  / SHARED_BITMAP_BITS_PER_WORD;
			unsigned long first_off  = first % SHARED_BITMAP_BITS_PER_WORD;
			unsigned long last_off   = last  % SHARED_BITMAP_BITS_PER_WORD;
			unsigned long w;

			if (first_word == last_word) {
				unsigned long width = last_off - first_off + 1;
				unsigned long mask  = (width == SHARED_BITMAP_BITS_PER_WORD)
					? ~0UL
					: (((1UL << width) - 1UL) << first_off);

				if (shared_region_bitmap[first_word] & mask)
					overlap = true;
			} else {
				unsigned long head_mask = ~0UL << first_off;

				if (shared_region_bitmap[first_word] & head_mask) {
					overlap = true;
				} else {
					for (w = first_word + 1; w < last_word; w++) {
						if (shared_region_bitmap[w]) {
							overlap = true;
							break;
						}
					}
					if (!overlap) {
						unsigned long tail_width = last_off + 1;
						unsigned long tail_mask  = (tail_width == SHARED_BITMAP_BITS_PER_WORD)
							? ~0UL
							: ((1UL << tail_width) - 1UL);

						if (shared_region_bitmap[last_word] & tail_mask)
							overlap = true;
					}
				}
			}
		}
	}

	/* Above-span / span-straddling queries cannot trust a bitmap
	 * negative: shared_bitmap_mark() no-ops for those addresses, so
	 * a registered region up there leaves no bitmap trace.  Force
	 * the candidate flag so the shared_regions[] linear scan below
	 * answers the membership question authoritatively. */
	if (addr >= SHARED_BITMAP_VA_SPAN ||
	    (len != 0 && len > SHARED_BITMAP_VA_SPAN - addr))
		overlap = true;

	if (!overlap)
		return false;

	/* Bitmap hit: confirm there is real byte-range overlap before
	 * rejecting.  This is the rarer path (bitmap-clear cases already
	 * returned above), so paying an O(N) walk here is cheap on the
	 * common fast path and only fires when the query genuinely
	 * shares a 2 MiB chunk with a tracked region. */
	if (!shared_regions_exact_overlap(addr, len, end))
		return false;

	child = this_child();
	if (child != NULL && child->stats_ring != NULL) {
		unsigned int nr = child->syscall.nr;
		bool do32 = child->syscall.do32bit;

		stats_ring_enqueue(child->stats_ring,
				   STATS_FIELD_RANGE_OVERLAPS_SHARED_REJECTS,
				   0, 1);
		if (nr < MAX_NR_SYSCALL) {
			enum stats_field f = do32
				? STATS_FIELD_RANGE_REJECTS_PER_SYSCALL_32
				: STATS_FIELD_RANGE_REJECTS_PER_SYSCALL_64;
			stats_ring_enqueue(child->stats_ring, f,
					   (uint16_t)nr, 1);
		}

		last_reject_syscall_nr = nr;
		last_reject_do32bit = do32 ? 1 : 0;
		last_reject_have_syscall = 1;
	} else {
		/* Parent / pre-fork context: bump the aggregate directly. */
		parent_stats.range_overlaps_shared_rejects++;
	}

	/* Per-process monotonic counter feeding the verbose rate-limited
	 * log below.  The canonical aggregate now lives in parent-private
	 * memory and is not directly visible from child context, so the
	 * "fleet-wide every Nth reject" cadence the original counter
	 * provided cannot be recovered cheaply.  Each child rate-limits
	 * its own log lines independently; the parent does the same.
	 * Verbosity-gated, informational only. */
	{
		static unsigned long local_n;

		n = ++local_n;
	}

	if (verbosity > 1 &&
	    (n % RANGE_OVERLAPS_SHARED_REJECT_REPORT_INTERVAL) == 0) {
		const char *sname = "?";
		unsigned int snr;
		unsigned char s32;

		if (last_reject_have_syscall) {
			snr = last_reject_syscall_nr;
			s32 = last_reject_do32bit;
			sname = print_syscall_name(snr, s32 != 0);
		}

		output(1, "range_overlaps_shared: %lu cumulative rejects "
			"(latest syscall=%s addr=0x%lx len=%lu)\n",
			n, sname, addr, len);
	}
	return true;
}

/*
 * Precise containment check: is [addr, addr+len) fully inside at least
 * one entry of shared_regions[]?  Used by get_writable_address() to
 * confirm a freshly-picked pool address still resolves to a tracked
 * mapping before handing it back to a sanitiser.
 *
 * Distinct from range_overlaps_shared() in three ways the caller
 * relies on:
 *   1. Polarity is "fully inside", not "overlaps".  A scribbled slot
 *      can hold a value that happens to abut a tracked region without
 *      being inside it; over-acceptance there would defeat the guard.
 *   2. Walks shared_regions[] linearly.  The bitmap accelerator that
 *      backs range_overlaps_shared() rounds to 2 MiB chunks, which is
 *      the SAFETY direction for a reject-shaped sanitiser but the
 *      WRONG direction here -- a 2 MiB chunk that contains some
 *      tracked region would falsely accept addresses elsewhere in the
 *      same chunk.
 *   3. Does not bump range_overlaps_shared_rejects.  This is a
 *      validation lookup, not a sanitiser reject; folding it into the
 *      reject counter would lie to the operator about how often the
 *      mm-syscall guards are firing.
 *
 * Empty ranges (len == 0) match if @addr lies strictly inside any
 * region; the caller controls len so this is the consistent shape.
 * Wrapped ranges return false (no real allocation can wrap user VA).
 */
bool range_in_tracked_shared(unsigned long addr, unsigned long len)
{
	unsigned long end;
	unsigned int i;

	if (len != 0 && addr > ULONG_MAX - len)
		return false;

	end = addr + len;

	for (i = 0; i < nr_shared_regions; i++) {
		unsigned long rstart = shared_regions[i].addr;
		unsigned long rend = rstart + shared_regions[i].size;

		if (addr >= rstart && end <= rend)
			return true;
	}
	/* Same byte-precise walk over the overflow tail: a region parked
	 * there is no less tracked from the caller's perspective, and a
	 * false negative would let get_writable_address() hand back a
	 * pool slot that no longer resolves to a tracked mapping. */
	for (i = 0; i < nr_shared_regions_overflow; i++) {
		unsigned long rstart = shared_regions_overflow[i].addr;
		unsigned long rend = rstart + shared_regions_overflow[i].size;

		if (addr >= rstart && end <= rend)
			return true;
	}
	return false;
}

void * __zmalloc(size_t size, const char *func)
{
	void *p;

	/*
	 * Tick the brk-cache refresh on the malloc path as well as the
	 * alloc_object() path.  Heavy __zmalloc users (cmp-hints /
	 * RedQueen pool inflation, per-syscall sequence records) can
	 * drive billions of malloc()s in a session without ever calling
	 * alloc_object(), and a malloc that triggers a brk grow leaves
	 * cached_brk_end behind by exactly that grow until the next
	 * alloc_object() refresh fires -- which on those workloads can
	 * be never.  Refreshing here closes the diagnostic-window race
	 * the heap_brk_stale_window_hit counter exists to measure.
	 */
	heap_brk_maybe_refresh();

	p = malloc(size);
	if (p == NULL) {
		/* Maybe we mlockall'd everything. Try and undo that, and retry. */
		munlockall();
		p = malloc(size);
		if (p != NULL)
			goto done;

		outputerr("%s: malloc(%zu) failure.\n", func, size);
		exit(EXIT_FAILURE);
	}

done:
	memset(p, 0, size);
	return p;
}

/*
 * Opt-in variant of __zmalloc() that additionally registers the
 * returned pointer with the deferred-free alloc-track ring.  Callers
 * use this when the allocation is bound to flow through
 * deferred_free_enqueue() / deferred_freeptr() so that the consume-on-
 * free invariant has a matching tracker entry to remove.  Plain
 * __zmalloc() must be used at sites whose allocations are released
 * via direct free() (process-lifetime / per-child tables / error-path
 * fallbacks); registering those would leave stale entries in the ring
 * that a fuzzed value can match against -- the bug Option B of the
 * 2026-05-19 alloc-tracking audit narrows the tracker to avoid.
 */
void * __zmalloc_tracked(size_t size, const char *func)
{
	void *p = __zmalloc(size, func);

	deferred_alloc_track(p, size);
	return p;
}

/*
 * Ownership table for syscall handlers that snapshot state into a
 * zmalloc'd struct hung off rec->post_state.  Currently only execve /
 * execveat use it, but the API is shape-agnostic so any post handler
 * that needs the same guarantee can call in.
 *
 * Background: rec->post_state is private to the post handler in the
 * syscall ABI sense, but the whole syscallrecord is reachable from
 * sibling fuzz writes -- a value-result write that lands on the
 * post_state slot can redirect it to a different, smaller heap
 * allocation that another syscall's own post_state owns.  The
 * post handler then copies sizeof(struct ...) bytes out of the foreign
 * chunk and trips an OOB read.
 *
 * The original guard against this was malloc_usable_size(snap) <
 * sizeof(*snap), which reads glibc's chunk-header allocation size.
 * That works under glibc but is undefined behaviour on a
 * non-malloc-owned pointer; libsanitizer treats it as a runtime error
 * and aborts the child with a SIGABRT cascade -- the guard meant to
 * catch sibling-stomp redirection becomes the new crash site under
 * ASAN.
 *
 * Replace the chunk-header probe with an explicit ownership table:
 * each handler registers its post_state pointer at allocation time and
 * unregisters before the deferred_freeptr() that releases it.  A snap
 * value that doesn't appear in the table cannot be a chunk we
 * produced, so the post handler bails without dereferencing.  The
 * lookup is pure pointer comparison -- well-defined under both glibc
 * and ASAN.
 *
 * Storage layout: 64-slot fixed pointer table in BSS, COW-shared at
 * fork, written single-threaded by the owning child.  No locking
 * needed.  Each child has at most one in-flight execve post_state at a
 * time (syscalls execute sequentially within a child), so the typical
 * working set is 0-1 entries; 64 slots leaves ample headroom for
 * collision tolerance and silent-drop on the rare table-full case.
 *
 * Hash: top bits of the pointer above glibc's 16-byte chunk
 * alignment.  Open addressing with linear probing for insert.  Lookup
 * and delete scan the table (bounded by POST_STATE_TABLE_SIZE) instead
 * of stopping at the first NULL slot, so a delete-induced gap can't
 * truncate a collision chain and leave a registered pointer
 * unreachable.  The scan cost is a couple of cache lines on the hot
 * path (per-syscall post handler) -- the typical hit lands at the
 * hash slot on the first probe.
 *
 * Scope: this is for the post_state ownership question specifically,
 * not a general validator for every __zmalloc() return.  Wrap the
 * allocation site at each interested caller rather than hooking
 * __zmalloc itself -- the vast majority of zmalloc callers don't need
 * this and the indirection cost would be wasted.
 */
#define POST_STATE_TABLE_SIZE	64
#define POST_STATE_TABLE_MASK	(POST_STATE_TABLE_SIZE - 1)

/*
 * Each slot carries the ptr plus a tag describing who installed the
 * snap, what magic word it carries, how large the allocation was, and
 * whether it has already been released.  The tag drives the
 * release-side rejection contract (wrong-owner / already-released /
 * untracked / bad-magic) so a sibling-stomped post_state can no longer
 * walk into libc free() and abort.
 *
 * Field semantics:
 *   ptr         - chunk address, or NULL if the slot is empty.
 *   syscall_nr  - rec->nr at install time, or UINT_MAX when the
 *                 installer used the untagged post_state_register()
 *                 entry point (legacy / non-canonical sites).  The
 *                 release path skips the wrong-owner check on
 *                 UINT_MAX so untagged sites retain prior behaviour
 *                 minus the abort-on-double-free.
 *   do32bit     - rec->do32bit at install time (paired with
 *                 syscall_nr to disambiguate the biarch table).
 *   magic       - leading-word cookie expected at *(unsigned long *)ptr.
 *                 Captured at install time from the freshly-stamped
 *                 snap[0]; check-static post-state-magic.sh enforces
 *                 that every post_state struct opens with `unsigned
 *                 long magic` so the read is well-defined.  Zero when
 *                 the installer used the untagged
 *                 post_state_register() entry point.
 *   size        - allocation size handed to zmalloc_tracked() at the
 *                 install site, threaded through for telemetry on
 *                 reject lines.  Zero when unknown; the release path
 *                 never gates on size (calling malloc_usable_size on a
 *                 stomped pointer would itself be UB under ASAN, the
 *                 same regression the ownership-table replaced).
 *   released    - flipped true by post_state_release() the first time
 *                 a snap is accepted for free; the second release call
 *                 on the same address sees released=true and rejects
 *                 (already-released) instead of double-freeing.  The
 *                 entry stays in the table with released=true until a
 *                 future post_state_register() probe lands on the slot
 *                 and overwrites it.
 */
struct post_state_entry {
	void *ptr;
	unsigned int syscall_nr;
	bool do32bit;
	bool released;
	unsigned long magic;
	size_t size;
};

static struct post_state_entry post_state_table[POST_STATE_TABLE_SIZE];

static unsigned int post_state_hash(const void *p)
{
	return (unsigned int) (((uintptr_t) p >> 4) & POST_STATE_TABLE_MASK);
}

/*
 * Locate the table entry for @p, including stale released entries so
 * post_state_release() can answer "already-released" specifically.
 * Returns NULL when no slot carries @p.  Scans the full table (the
 * unregister path leaves holes mid-chain) so a NULL slot in the
 * collision chain does not truncate the search.
 */
static struct post_state_entry *post_state_table_find(const void *p)
{
	unsigned int idx;
	unsigned int i;

	if (p == NULL)
		return NULL;

	idx = post_state_hash(p);
	for (i = 0; i < POST_STATE_TABLE_SIZE; i++) {
		if (post_state_table[idx].ptr == p)
			return &post_state_table[idx];
		idx = (idx + 1) & POST_STATE_TABLE_MASK;
	}
	return NULL;
}

/*
 * Insert @p with the supplied tag.  Reuses slots that are empty or
 * carry a released==true entry (the chunk those described is gone, the
 * slot is a stale telemetry record).  An idempotent re-insert of @p at
 * a still-live slot is a no-op.  Table full → silently drop the
 * registration; lookup will miss, the post handler will bail without
 * dereferencing, the chunk leaks until child exit (benign).
 */
static void post_state_register_full(void *p, unsigned int syscall_nr,
				     bool do32bit, unsigned long magic,
				     size_t size)
{
	unsigned int idx;
	unsigned int i;

	if (p == NULL)
		return;

	idx = post_state_hash(p);
	for (i = 0; i < POST_STATE_TABLE_SIZE; i++) {
		if (post_state_table[idx].ptr == NULL ||
		    post_state_table[idx].released) {
			post_state_table[idx].ptr = p;
			post_state_table[idx].syscall_nr = syscall_nr;
			post_state_table[idx].do32bit = do32bit;
			post_state_table[idx].released = false;
			post_state_table[idx].magic = magic;
			post_state_table[idx].size = size;
			return;
		}
		if (post_state_table[idx].ptr == p)
			return;
		idx = (idx + 1) & POST_STATE_TABLE_MASK;
	}
}

void post_state_register(void *p)
{
	post_state_register_full(p, UINT_MAX, false, 0, 0);
}

void post_state_unregister(void *p)
{
	unsigned int idx;
	unsigned int i;

	if (p == NULL)
		return;

	idx = post_state_hash(p);
	for (i = 0; i < POST_STATE_TABLE_SIZE; i++) {
		if (post_state_table[idx].ptr == p) {
			post_state_table[idx].ptr = NULL;
			post_state_table[idx].syscall_nr = 0;
			post_state_table[idx].do32bit = false;
			post_state_table[idx].released = false;
			post_state_table[idx].magic = 0;
			post_state_table[idx].size = 0;
			return;
		}
		idx = (idx + 1) & POST_STATE_TABLE_MASK;
	}
}

bool post_state_is_owned(const void *p)
{
	struct post_state_entry *e = post_state_table_find(p);

	return e != NULL && !e->released;
}

/*
 * Tagged install: the canonical entry point invoked via the
 * post_state_install() macro in include/utils.h, which supplies @size
 * as sizeof(*snap) at the call site.  Captures the snap's magic word
 * from snap[0] at install time -- by post-state-magic.sh convention
 * every post_state struct opens with `unsigned long magic` and the
 * .sanitise body must stamp it BEFORE calling this helper, so the
 * leading-word read aliases snap->magic without needing the caller's
 * struct type.  Storing the magic in the ownership table closes the
 * "snap chunk was freed and reallocated under us by another path"
 * window the release-side reject contract checks against.
 */
void post_state_install_sized(struct syscallrecord *rec, void *snap,
			      size_t size)
{
	unsigned long magic = 0;

	rec->post_state = (unsigned long) snap;
	if (snap != NULL)
		magic = *(const unsigned long *) snap;
	post_state_register_full(snap, rec->nr, rec->do32bit, magic, size);
}

/*
 * Canonical .post-entry gate.  See the helper-block comment in
 * include/utils.h for the rationale and the three-step ordering this
 * encodes.  Diagnostic strings and counter bumps mirror the prior
 * hand-rolled gates so log readers and stat dashboards keep working.
 *
 * The shape gate calls looks_like_corrupted_ptr_pc() directly with the
 * caller PC fetched by __builtin_return_address(0); that preserves the
 * per-callsite PC attribution that the static-inline looks_like_corrupted_ptr()
 * wrapper would otherwise lose now that we are a separate function.
 *
 * The magic word is read via *(const unsigned long *)snap rather than
 * snap->magic; every post_state struct puts `unsigned long magic`
 * first by convention (post-state-magic.sh enforces it), so the
 * leading-word read aliases the magic field without the helper needing
 * to know the caller's struct type.
 */
void *post_state_claim_owned(struct syscallrecord *rec,
			     unsigned long magic_expected,
			     const char *handler_name)
{
	void *snap = (void *) rec->post_state;
	unsigned long magic_found;

	if (snap == NULL)
		return NULL;

	if (looks_like_corrupted_ptr_pc(rec, snap, __builtin_return_address(0))) {
		outputerr("%s: rejected suspicious post_state=%p "
			  "(pid-scribbled?)\n", handler_name, snap);
		rec->post_state = 0;
		return NULL;
	}

	/*
	 * Ownership gate -- MUST run before reading any field of snap,
	 * including the magic cookie below.  A foreign chunk that survived
	 * the shape gate may not even be sizeof(unsigned long) bytes in
	 * size; reading the leading word on a non-snap allocation is a
	 * wild read.  post_state_is_owned() is pure pointer comparison
	 * against the table and is well-defined regardless of what snap
	 * points at.
	 */
	if (!post_state_is_owned(snap)) {
		outputerr("%s: rejected post_state=%p not in ownership table "
			  "(post_state-redirected?)\n", handler_name, snap);
		post_handler_corrupt_ptr_bump_at(rec, NULL,
						 CORRUPT_PTR_SITE_CLAIM_OWNED_NOT_OWNED);
		rec->post_state = 0;
		return NULL;
	}

	/*
	 * Ownership confirmed -- snap really is one of our chunks, so
	 * reading the leading word is now safe.  By convention every
	 * post_state struct puts `unsigned long magic` first, so this read
	 * aliases snap->magic without a typed deref.
	 */
	magic_found = *(const unsigned long *) snap;
	if (magic_found != magic_expected) {
		outputerr("%s: rejected snap with bad magic 0x%lx "
			  "(post_state-stomped to foreign allocation?)\n",
			  handler_name, magic_found);
		post_handler_corrupt_ptr_bump_at(rec, NULL,
						 CORRUPT_PTR_SITE_CLAIM_OWNED_BAD_MAGIC);
		rec->post_state = 0;
		return NULL;
	}

	return snap;
}

/*
 * Idempotent release contract.  Every rejection path leaves the chunk
 * alive (no libc free()), clears rec->post_state so a second .post /
 * .cleanup invocation on the same rec can no longer rediscover the
 * stale pointer, and bumps a structured counter so the rate is
 * visible without grepping outputerr lines.  The four rejections,
 * checked in order:
 *
 *   1. untracked   - snap is not in the ownership table.  Either it was
 *      never registered (pure sibling stomp landed on rec->post_state),
 *      or its table slot was already overwritten by a later
 *      registration after a prior release.  Both shapes mean we have
 *      no proof @snap is a real malloc-returned chunk; handing it to
 *      free() is the libc abort the spec asked us to stop.
 *
 *   2. already-released - snap is in the table but a prior
 *      post_state_release() already accepted it.  The .post and
 *      .cleanup helpers both route through here, and an .post that
 *      released followed by a .cleanup that releases the same snap
 *      would currently double-free.  Idempotence drops the second
 *      call without touching libc.
 *
 *   3. wrong-owner - snap is live, but the installer was a different
 *      (syscall_nr, do32bit) than the caller.  This is the
 *      sibling-stomp redirect class: another handler's snap got
 *      pointed at by our rec->post_state via a fuzzed value-result
 *      write; we would otherwise free the other handler's chunk and
 *      it would crash when its own .post handler tries to read it
 *      back.  Untagged installers (post_state_register-only,
 *      syscall_nr==UINT_MAX) skip this gate so legacy sites retain
 *      prior behaviour modulo the abort-on-double-free.
 *
 *   4. bad-magic - snap is live and tagged to us, but the leading
 *      word no longer matches the magic captured at install time.
 *      The chunk's contents have been overwritten by something that
 *      is not our post_state snap; freeing it would be freeing
 *      something we no longer own.  Untagged installers (magic==0)
 *      skip this gate.
 *
 * Only after all four gates pass do we mark the entry released and
 * hand the chunk to deferred_freeptr(), which performs its own
 * shape / heap-bounds / alloc_track / shared-region cascade as the
 * second wall.
 */
void post_state_release(struct syscallrecord *rec, void *snap)
{
	struct post_state_entry *e;
	unsigned long magic_found;

	if (snap == NULL)
		return;

	e = post_state_table_find(snap);
	if (e == NULL) {
		outputerr("post_state_release: rejected untracked snap=%p "
			  "(caller nr=%u do32bit=%d) -- leaking, not freeing\n",
			  snap, rec->nr, rec->do32bit);
		__atomic_add_fetch(&shm->stats.post_state_release_reject_untracked,
				   1, __ATOMIC_RELAXED);
		rec->post_state = 0;
		return;
	}

	if (e->released) {
		outputerr("post_state_release: rejected already-released snap=%p "
			  "(prior owner nr=%u do32bit=%d, caller nr=%u do32bit=%d)\n",
			  snap, e->syscall_nr, e->do32bit, rec->nr, rec->do32bit);
		__atomic_add_fetch(&shm->stats.post_state_release_reject_released,
				   1, __ATOMIC_RELAXED);
		rec->post_state = 0;
		return;
	}

	if (e->syscall_nr != UINT_MAX &&
	    (e->syscall_nr != rec->nr || e->do32bit != rec->do32bit)) {
		outputerr("post_state_release: rejected wrong-owner snap=%p "
			  "(owner nr=%u do32bit=%d, caller nr=%u do32bit=%d, "
			  "size=%zu) -- leaking, not freeing\n",
			  snap, e->syscall_nr, e->do32bit, rec->nr, rec->do32bit,
			  e->size);
		__atomic_add_fetch(&shm->stats.post_state_release_reject_wrong_owner,
				   1, __ATOMIC_RELAXED);
		rec->post_state = 0;
		return;
	}

	if (e->magic != 0) {
		magic_found = *(const unsigned long *) snap;
		if (magic_found != e->magic) {
			outputerr("post_state_release: rejected snap=%p bad magic "
				  "(found 0x%lx, expected 0x%lx, owner nr=%u) -- "
				  "leaking, not freeing\n",
				  snap, magic_found, e->magic, e->syscall_nr);
			__atomic_add_fetch(&shm->stats.post_state_release_reject_bad_magic,
					   1, __ATOMIC_RELAXED);
			rec->post_state = 0;
			return;
		}
	}

	e->released = true;
	deferred_freeptr(&rec->post_state);
}

void sizeunit(unsigned long size, char *buf, size_t buflen)
{
	/* non kilobyte aligned size? */
	if (size < 1024) {
		snprintf(buf, buflen, "%lu bytes", size);
		return;
	}

	/* < 1MB ? */
	if (size < (1024 * 1024)) {
		snprintf(buf, buflen, "%luKB", size / 1024);
		return;
	}

	/* < 1GB ? */
	if (size < (1024 * 1024 * 1024)) {
		snprintf(buf, buflen, "%luMB", (size / 1024) / 1024);
		return;
	}

	snprintf(buf, buflen, "%luGB", ((size / 1024) / 1024) / 1024);
}

void kill_pid(pid_t pid)
{
	int ret;
	int childno;

	if (pid == -1) {
		show_backtrace();
		syslogf("kill_pid tried to kill -1!\n");
		return;
	}
	if (pid == 0) {
		show_backtrace();
		syslogf("tried to kill_pid 0!\n");
		return;
	}

	/*
	 * Refuse to SIGKILL ourselves.  A wrapper run was observed dying
	 * with mainpid SIGKILL'ing mainpid; bpftrace on signal_generate
	 * confirmed the kill syscall came from main itself.  The path is
	 * shm corruption scribbling mainpid into a pids[] slot, then a
	 * reap/kill loop walking pids[] and feeding that value back in
	 * here.  pid_is_valid() accepts mainpid as in-range, so without
	 * this guard main commits suicide.
	 *
	 * Scan pids[] first so the diagnostic line names the scribbled
	 * slot, then dump childnos + the pids page state so we can tell
	 * whether this was a single wild write or a page-level event.
	 */
	if (pid == mainpid) {
		unsigned int i;
		int corrupt_slot = -1;

		for_each_child(i) {
			pid_t slot = __atomic_load_n(&pids[i], __ATOMIC_RELAXED);
			if (slot == mainpid) {
				corrupt_slot = i;
				break;
			}
		}

		if (corrupt_slot == -1)
			syslogf("kill_pid refused: pid=%d == mainpid=%d, pids[] slot=none\n",
				pid, mainpid);
		else
			syslogf("kill_pid refused: pid=%d == mainpid=%d, pids[] slot=%d\n",
				pid, mainpid, corrupt_slot);

		show_backtrace();
		dump_childnos();
		dump_pids_page_state();
		return;
	}

	childno = find_childno(pid);
	if (childno != CHILD_NOT_FOUND) {
		if (children[childno]->dontkillme == true)
			return;
	}

	ret = kill(pid, SIGKILL);
	if (ret != 0)
		debugf("couldn't kill pid %d [%s]\n", pid, strerror(errno));
}

/*
 * looks_like_corrupted_ptr - heuristic test for "this slot used to hold
 * a pointer we malloc'd, but somebody scribbled a non-pointer over it".
 *
 * Cluster-1 / cluster-2 / cluster-3 crash signature (residual-cores
 * triage 2026-05-02): si_addr in the killing siginfo equals si_pid (e.g.
 * si_addr=0x378a02 against pid 0x378a02).  The shape comes from a fuzzed
 * value-result syscall in some sibling child landing in trinity-internal
 * memory -- rec->aN, a struct field reachable from rec->aN, or a slot in
 * the deferred-free ring -- and overwriting a pointer that a post handler
 * was about to deref or pass to free(), with the kernel-issued tid/pid
 * value.  The deferred-free ring already mprotects between ticks so a
 * scribble there now SIGSEGVs in copy_from_user, but the rec-> path is
 * unprotected by construction (the kernel must be able to write into
 * rec->aN -- that's the whole point).
 *
 * Three rejection bands, all heuristic:
 *
 *   - v < 0x10000:  cannot be a real heap pointer.  PIDs (pid_max is
 *     typically 4 million on Linux) and small ints land here.  This is
 *     the same gate deferred_free_tick() uses as a belt-and-braces
 *     check at free time; we want to reject earlier so the ring slot
 *     never holds a bogus value at all.
 *
 *   - v >= (1UL << 47):  above the x86_64 user canonical limit.  glibc
 *     malloc / mmap / brk hand out addresses well below this; a value
 *     here is either a kernel pointer leaked back (bug regardless of
 *     post-handler state) or a tornadic write of a high-bit pattern.
 *
 *   - v & 0x7:  misaligned for an 8-byte pointer.  Every trinity heap
 *     allocator (zmalloc, alloc_iovec, get_writable_*, alloc_object)
 *     hands back >= 8-byte aligned memory.  A misaligned value in a
 *     slot we expect to free is almost certainly a partial overwrite.
 *
 * False positive cost: a legitimate-but-weird pointer would be dropped
 * (memory leak), not freed.  A leak in a post handler is benign --
 * children turn over fast and the heap evaporates at exit.  The
 * alternative (false negative) is the cluster-1/2/3 SIGSEGV class we
 * are trying to kill, so we err strict.  Audited against current post
 * handlers (deferred_freeptr, deferred_free_enqueue callers, direct
 * free() on rec->aN): every pointer those receive is heap-allocated
 * via 8-byte aligned routines, so the misalign band is safe at all
 * present call sites.  If a future caller is shown to legitimately
 * pass an unaligned value, drop the alignment check rather than
 * dropping the others.
 *
 * Returns true if the pointer looks scribbled and the caller should
 * drop it instead of dereferencing or freeing.
 */
/*
 * Update this child's per-handler attribution shard for a
 * post_handler_corrupt_ptr rejection.  Linear scan of the 32-entry shard:
 * if @nr is already present we bump its count; otherwise the lowest-count
 * slot is evicted in favour of the new key.  Eviction stays child-local,
 * so global LRU ordering across the fleet is lost on the long tail; the
 * parent merges every child's shard at dump time and the hot handlers
 * still surface in the top rows because they land in every shard.
 *
 * Each child is the sole writer of its own shard, so no lock is needed.
 * The parent is the sole reader and only at periodic-dump time -- torn
 * reads on the dump side may shave a count by one off a single shard
 * slot, which is in the noise once 32 shards are merged.
 *
 * this_child()==NULL callers (parent post-mortem paths, deferred-free
 * tick on the main process) have no shard to bump and drop the record.
 */
static void corrupt_ptr_attr_record(unsigned int nr, bool do32bit)
{
	struct childdata *child = this_child();
	struct corrupt_ptr_attr_entry *ring;
	unsigned int i, victim;
	unsigned long victim_count;

	if (child == NULL)
		return;

	ring = child->local_corrupt_ptr_attr;

	for (i = 0; i < CORRUPT_PTR_ATTR_SLOTS; i++) {
		if (ring[i].count != 0 &&
		    ring[i].nr == nr && ring[i].do32bit == do32bit) {
			ring[i].count++;
			return;
		}
	}

	victim = 0;
	victim_count = ring[0].count;
	for (i = 1; i < CORRUPT_PTR_ATTR_SLOTS; i++) {
		if (ring[i].count < victim_count) {
			victim = i;
			victim_count = ring[i].count;
		}
		if (victim_count == 0)
			break;
	}

	ring[victim].nr = nr;
	ring[victim].do32bit = do32bit;
	ring[victim].count = victim_count + 1;
}

/*
 * Record a (nr, do32bit, pc) triple into this child's per-callsite
 * sub-attribution shard.  Same eviction policy as corrupt_ptr_attr_record:
 * bump the matching slot if present, otherwise displace the lowest-count
 * slot.  No lock for the same reason -- the owning child is the sole
 * writer.  Skipped when pc==NULL (defensive -- a caller without a usable
 * return address has no useful PC to record) or when this_child() returns
 * NULL (no shard to bump).
 */
static void corrupt_ptr_pc_record(unsigned int nr, bool do32bit, void *pc,
				  const char *site)
{
	struct childdata *child = this_child();
	struct corrupt_ptr_pc_entry *ring;
	unsigned int i, victim;
	unsigned long victim_count;

	if (pc == NULL || child == NULL)
		return;

	ring = child->local_corrupt_ptr_pc;

	for (i = 0; i < CORRUPT_PTR_PC_SLOTS; i++) {
		if (ring[i].count != 0 &&
		    ring[i].nr == nr && ring[i].do32bit == do32bit &&
		    ring[i].pc == pc) {
			ring[i].count++;
			/* Late-arriving site tag for an existing entry — fill
			 * it in so the dump can disambiguate even when the
			 * first bump for this PC came through a tagless caller
			 * (e.g. the legacy macro wrapper with site=NULL). */
			if (ring[i].site == NULL && site != NULL)
				ring[i].site = site;
			return;
		}
	}

	victim = 0;
	victim_count = ring[0].count;
	for (i = 1; i < CORRUPT_PTR_PC_SLOTS; i++) {
		if (ring[i].count < victim_count) {
			victim = i;
			victim_count = ring[i].count;
		}
		if (victim_count == 0)
			break;
	}

	ring[victim].nr = nr;
	ring[victim].do32bit = do32bit;
	ring[victim].pc = pc;
	ring[victim].site = site;
	ring[victim].count = victim_count + 1;
}

void post_handler_corrupt_ptr_bump_full(struct syscallrecord *rec,
					void *caller_pc, const char *site,
					unsigned int arg_idx,
					unsigned long bad_ptr)
{
	struct childdata *child;
	unsigned int nr;
	bool do32bit;

	/* Headline aggregate routes through the per-child stats_ring on
	 * the child path (parent drain accumulates into parent_stats).
	 * Parent-context callers (post-mortem paths, deferred-free tick
	 * on the main process) bump parent_stats directly since the
	 * parent is the sole writer in that case. */
	child = this_child();
	if (child != NULL && child->stats_ring != NULL)
		stats_ring_enqueue(child->stats_ring,
				   STATS_FIELD_POST_HANDLER_CORRUPT_PTR, 0, 1);
	else
		parent_stats.post_handler_corrupt_ptr++;

	/* Per-child shadow of the same event, scored by the storm-rate
	 * check in child_process.  Stays per-child (not in shm) -- the
	 * storm check reads it directly off childdata. */
	if (child != NULL)
		child->local_post_handler_corrupt_ptr++;

	if (rec != NULL) {
		nr = rec->nr;
		do32bit = rec->do32bit;
	} else {
		nr = CORRUPT_PTR_ATTR_NR_NONE;
		do32bit = false;
	}
	corrupt_ptr_pc_record(nr, do32bit, caller_pc, site);
	corrupt_ptr_attr_record(nr, do32bit);

	/* Per-fire breadcrumb with the scribbled pointer value, the arg
	 * slot the caller attributed it to (or NO_ARG), and the site tag.
	 * Drops silently when this_child() is NULL -- per-child storage. */
	corrupt_ptr_breadcrumb_push(rec, arg_idx, bad_ptr, site);
}

void post_handler_corrupt_ptr_bump_site(struct syscallrecord *rec,
					void *caller_pc, const char *site)
{
	post_handler_corrupt_ptr_bump_full(rec, caller_pc, site,
					   CORRUPT_PTR_BREADCRUMB_NO_ARG,
					   CORRUPT_PTR_BREADCRUMB_BAD_UNKNOWN);
}

/*
 * TRINITY_CORRUPT_ATTRIB measurement path.  Off by default; enabled at
 * runtime by exporting TRINITY_CORRUPT_ATTRIB=1 before fork.  Latched
 * on first query so the hot path pays one cached-bool branch when the
 * gate is off.
 *
 * The mprotect-armor probe (corruption probe #1) proved that the
 * syscallrecord itself is not wild-written (0 trap fires, rec_canary=0)
 * yet the post_handler_corrupt_ptr headline keeps spiking ~40k/60s.
 * This counter must therefore be conflating distinct rejection paths
 * -- structural validators (validate_arg_coupling, enforce_count_bound)
 * that bump on perfectly-fine-but-rejected kernel results AND the
 * shape-heuristic firing on genuine scribbles into rec->aN AND the
 * per-handler oracle bumps (mq_notify / getitimer / timer_gettime /
 * timerfd_gettime).  Per-site attribution disambiguates: if the spike
 * is dominated by validator_rejected the headline tells us nothing
 * about real corruption; if a residual "post_generic" bucket remains
 * non-trivial, that's the next call-site sweep target.
 */
_Static_assert(sizeof(((struct stats_s *)0)->corrupt_ptr_site_count) ==
	       sizeof(unsigned long) * CORRUPT_PTR_SITE__COUNT,
	       "corrupt_ptr_site_count array size out of sync with enum");

const char *const corrupt_ptr_site_names[CORRUPT_PTR_SITE__COUNT] = {
	[CORRUPT_PTR_SITE_VALIDATOR_REJECTED]    = "validator_rejected",
	[CORRUPT_PTR_SITE_ENFORCE_COUNT_BOUND]   = "enforce_count_bound",
	[CORRUPT_PTR_SITE_RETFD_INVALID]         = "retfd_invalid",
	[CORRUPT_PTR_SITE_CLAIM_OWNED_NOT_OWNED] = "claim_owned_not_owned",
	[CORRUPT_PTR_SITE_CLAIM_OWNED_BAD_MAGIC] = "claim_owned_bad_magic",
	[CORRUPT_PTR_SITE_SHAPE_HEURISTIC]       = "shape_heuristic",
	[CORRUPT_PTR_SITE_MQ_NOTIFY]             = "mq_notify",
	[CORRUPT_PTR_SITE_GETITIMER]             = "getitimer",
	[CORRUPT_PTR_SITE_TIMER_GETTIME]         = "timer_gettime",
	[CORRUPT_PTR_SITE_TIMERFD_GETTIME]       = "timerfd_gettime",
};

static bool corrupt_attrib_inited;
static bool corrupt_attrib_enabled;

bool corrupt_ptr_attrib_active(void)
{
	if (!corrupt_attrib_inited) {
		const char *v = getenv("TRINITY_CORRUPT_ATTRIB");

		corrupt_attrib_enabled =
			(v != NULL && v[0] == '1' && v[1] == '\0');
		corrupt_attrib_inited = true;
	}
	return corrupt_attrib_enabled;
}

void corrupt_ptr_site_record(enum corrupt_ptr_site site)
{
	if (!corrupt_ptr_attrib_active())
		return;
	if ((unsigned int) site >= CORRUPT_PTR_SITE__COUNT)
		return;
	__atomic_add_fetch(&shm->stats.corrupt_ptr_site_count[site], 1,
			   __ATOMIC_RELAXED);
}

void post_handler_corrupt_ptr_bump_at(struct syscallrecord *rec,
				      void *caller_pc,
				      enum corrupt_ptr_site site)
{
	const char *tag = NULL;

	if ((unsigned int) site < CORRUPT_PTR_SITE__COUNT)
		tag = corrupt_ptr_site_names[site];
	corrupt_ptr_site_record(site);
	post_handler_corrupt_ptr_bump_site(rec, caller_pc, tag);
}

/*
 * Record a caller PC into this child's deferred_free_reject sub-attribution
 * shard.  Same eviction policy and ownership model as corrupt_ptr_pc_record
 * -- the owning child is the sole writer of its own shard, so no lock is
 * needed; the parent merges every child's shard at dump time.  Skipped when
 * pc==NULL (defensive -- a caller without a usable return address has no
 * useful PC to record) or when this_child() returns NULL (parent post-mortem
 * path, deferred-free tick on the main process -- no shard to bump).
 * Slimmer key than corrupt_ptr_pc_record because every bump originates from
 * rec==NULL deferred_free_enqueue calls so (nr, do32bit) carry no
 * information.
 */
static void deferred_free_reject_pc_record(void *pc)
{
	struct childdata *child = this_child();
	struct deferred_free_reject_pc_entry *ring;
	unsigned int i, victim;
	unsigned long victim_count;

	if (pc == NULL || child == NULL)
		return;

	ring = child->local_deferred_free_reject_pc;

	for (i = 0; i < CORRUPT_PTR_PC_SLOTS; i++) {
		if (ring[i].count != 0 && ring[i].pc == pc) {
			ring[i].count++;
			return;
		}
	}

	victim = 0;
	victim_count = ring[0].count;
	for (i = 1; i < CORRUPT_PTR_PC_SLOTS; i++) {
		if (ring[i].count < victim_count) {
			victim = i;
			victim_count = ring[i].count;
		}
		if (victim_count == 0)
			break;
	}

	ring[victim].pc = pc;
	ring[victim].count = victim_count + 1;
}

void deferred_free_reject_bump(void *caller_pc)
{
	struct childdata *child = this_child();
	enum stats_field shard;

	switch (deferred_free_get_cleanup_argtype()) {
	case ARG_PATHNAME:
		shard = STATS_FIELD_DEFERRED_FREE_REJECT_PATHNAME;
		break;
	case ARG_IOVEC:
	case ARG_IOVEC_IN:
		shard = STATS_FIELD_DEFERRED_FREE_REJECT_IOVEC;
		break;
	case ARG_SOCKADDR:
		shard = STATS_FIELD_DEFERRED_FREE_REJECT_SOCKADDR;
		break;
	default:
		shard = STATS_FIELD_DEFERRED_FREE_REJECT_OTHER;
		break;
	}

	if (child != NULL && child->stats_ring != NULL) {
		stats_ring_enqueue(child->stats_ring,
				   STATS_FIELD_DEFERRED_FREE_REJECT, 0, 1);
		stats_ring_enqueue(child->stats_ring, shard, 0, 1);
	} else {
		parent_stats.deferred_free_reject++;
		switch (shard) {
		case STATS_FIELD_DEFERRED_FREE_REJECT_PATHNAME:
			parent_stats.deferred_free_reject_pathname++;
			break;
		case STATS_FIELD_DEFERRED_FREE_REJECT_IOVEC:
			parent_stats.deferred_free_reject_iovec++;
			break;
		case STATS_FIELD_DEFERRED_FREE_REJECT_SOCKADDR:
			parent_stats.deferred_free_reject_sockaddr++;
			break;
		default:
			parent_stats.deferred_free_reject_other++;
			break;
		}
	}
	deferred_free_reject_pc_record(caller_pc);
}

__attribute__((noinline))
void post_handler_corrupt_ptr_bump_retfd(struct syscallrecord *rec)
{
	corrupt_ptr_site_record(CORRUPT_PTR_SITE_RETFD_INVALID);
	post_handler_corrupt_ptr_bump_site(rec, __builtin_return_address(0),
					   "handle_syscall_ret:retfd_invalid");
}

/*
 * Out-of-line tripwire for get_arg_snapshot() mismatches.  Called only
 * when an opted-in slot's shadow disagrees with the live rec->aN at the
 * post handler's read site, i.e. a sibling scribbled the slot after the
 * dispatch-time snapshot in __do_syscall() (taken from the local a1..a6
 * just before the syscall is issued) and before the post handler ran.
 * The narrower window is intentional: a stomp earlier than dispatch
 * was seen by the kernel directly and isn't a post-handler bound
 * fabrication, so it does not belong in this counter.
 * Routes through the per-child stats_ring on the child path (parent
 * drain accumulates into parent_stats.arg_shadow_stomp) and through
 * parent_stats directly on the rare no-child path.  No per-syscall
 * shard for now -- the MVP opted-in set is small enough that aggregate
 * tells us "is this signal real" without sub-attribution; when the
 * opt-in set grows past a handful of handlers we can teach this helper
 * the corrupt_ptr_attr_record per-(nr,do32bit) routing.
 */
void arg_shadow_stomp_bump(struct syscallrecord *rec, unsigned int argnum,
			   unsigned long shadow, unsigned long current)
{
	struct childdata *child = this_child();

	if (child != NULL && child->stats_ring != NULL)
		stats_ring_enqueue(child->stats_ring,
				   STATS_FIELD_ARG_SHADOW_STOMP, 0, 1);
	else
		parent_stats.arg_shadow_stomp++;

	output(0, "arg_shadow_stomp: syscall nr %u arg %u shadow 0x%lx live 0x%lx\n",
	       rec != NULL ? rec->nr : 0, argnum, shadow, current);
}

/*
 * Categorise a rejected pointer value into one of four heuristic bands
 * so the sample log line tells us at a glance whether the rejection is
 * obvious noise (NULL-ish, pid-shaped, kernel-VA leak) or whether the
 * value sits in the heap-shape range and the shape heuristic itself is
 * the false positive.  The pid_max ceiling of 4194304 (the kernel
 * default for 64-bit boots) is used for the pid-shaped band so a stray
 * tid lands here even though it would also satisfy v >= 0x10000.
 */
const char *corrupt_ptr_label(unsigned long v)
{
	if (v < 0x10000)
		return "NULL-ish";
	if (v < 4194304)
		return "pid-shaped";
	if (v >= 0x800000000000UL)
		return "kernel-VA";
	return "heap-shaped";
}

/*
 * Sample-rate for the per-rejection log line.  At the observed sustained
 * rate of ~900 rejections/min a 1-in-100 sample emits roughly nine lines
 * per minute -- enough to characterise the value distribution without
 * flooding the log faster than the operator can read it.
 */
#define CORRUPT_PTR_SAMPLE_INTERVAL	100

bool looks_like_corrupted_ptr_pc(struct syscallrecord *rec, const void *p,
				 void *caller_pc)
{
	unsigned long v = (unsigned long) p;
	unsigned long n;

	if (!is_corrupt_ptr_shape(p))
		return false;

	corrupt_ptr_site_record(CORRUPT_PTR_SITE_SHAPE_HEURISTIC);
	post_handler_corrupt_ptr_bump_full(rec, caller_pc, "shape-heuristic",
					   CORRUPT_PTR_BREADCRUMB_NO_ARG, v);

	/*
	 * Sample every CORRUPT_PTR_SAMPLE_INTERVALth rejection.  Counter
	 * is shm-resident so the sample cadence is fleet-global rather than
	 * per-child -- a host with 32 children would otherwise emit 32x the
	 * sample volume.  RELAXED ordering: the sample is opportunistic;
	 * losing one to a torn read between siblings does not matter.
	 */
	n = __atomic_add_fetch(&shm->stats.corrupt_ptr_sample_seq, 1,
			       __ATOMIC_RELAXED);
	if ((n % CORRUPT_PTR_SAMPLE_INTERVAL) == 1) {
		const char *name;
		char pcbuf[128];

		if (rec != NULL)
			name = print_syscall_name(rec->nr, rec->do32bit);
		else
			name = "<deferred-free>";

		output(0, "corrupt-ptr reject sample: syscall=%s value=0x%lx "
			  "label=%s caller=%s\n",
			  name, v, corrupt_ptr_label(v),
			  pc_to_string(__builtin_return_address(0),
				       pcbuf, sizeof(pcbuf)));
	}
	return true;
}

bool inner_ptr_ok_to_free(struct syscallrecord *rec, const void *p,
			  const char *site)
{
	unsigned long v = (unsigned long) p;

	if (p == NULL)
		return false;

	if (!looks_like_corrupted_ptr(rec, p))
		return true;

	/*
	 * Heap-shaped but misaligned -- the exact band that bypasses the
	 * outer-ptr alignment guard (because the outer ptr passes) but
	 * trips libasan's PoisonShadow CHECK once the inner field reaches
	 * free().  Surface it explicitly so log scanners can correlate the
	 * interception with the asan_poisoning.cpp:37 crash signature; the
	 * per-handler attribution ring already names the syscall via rec.
	 */
	if (v >= 0x10000 && (v & 0x7) != 0)
		outputerr("%s: rejected misaligned heap-shaped inner ptr=0x%lx "
			  "(libasan PoisonShadow trigger; scribbled?)\n",
			  site, v);
	return false;
}

/*
 * Cached extent of the brk()-managed glibc arena, captured once at
 * init time from /proc/self/maps.  COW-shared into every forked
 * child, so a single pre-fork parse seeds the whole fleet.  heap_start
 * is stable for the lifetime of the process (the brk base doesn't
 * move), but heap_end is only a snapshot -- a child that extends brk
 * post-fork (millions of iterations of getline in /proc parsers,
 * libasan shadow growth, heavy zmalloc traffic) sails past it, so
 * consumers also consult sbrk(0) for the current break.  Zero start
 * means we never found a [heap] line; in that case is_in_glibc_heap()
 * falls back to "always true" so we don't reject legitimate frees on
 * platforms or builds where glibc has chosen an mmap-only allocation
 * strategy and the brk arena is empty.
 */
static unsigned long heap_start;
static unsigned long heap_end;

/*
 * Per-child cached snapshot of sbrk(0), refreshed at heap_bounds_init()
 * time and then on every BRK_REFRESH_INTERVAL alloc_object() call via
 * heap_brk_maybe_refresh().  Used by is_in_glibc_heap() and
 * range_overlaps_libc_heap() instead of a per-call sbrk(0) -- those
 * gates fire on every deferred-free check and every random-address
 * generation, and the syscall-per-call was showing up in profiles.
 *
 * Staleness is bounded by the refresh cadence: in the worst case a
 * brk grow that happens between refreshes is invisible until the next
 * refresh fires, so a fuzzed pointer that lands in the extension is
 * not detected as heap-internal during that window.  brk grows in
 * page-or-larger chunks at a much lower rate than alloc_object() is
 * called (most allocations are served from the existing arena), so
 * the window is small in absolute address space.  Refresh tied to
 * alloc_object() means a brk grow caused by an allocation gets caught
 * by the next refresh that fires, and refresh-on-zero ensures
 * pre-init callers see "unknown extent" rather than a stale value.
 */
#define BRK_REFRESH_INTERVAL 64
static unsigned long cached_brk_end;
static unsigned int brk_refresh_counter;

/*
 * Slack ceiling above the cached brk upper bound for the
 * self-correcting near-edge re-test in is_in_glibc_heap() and
 * range_overlaps_libc_heap().  An address that sits in
 * [cached_upper, cached_upper + SLACK) is plausibly in a brk
 * extension that grew between BRK_REFRESH_INTERVAL ticks (heavy
 * non-alloc_object() allocator traffic can outrun the alloc-driven
 * refresh cadence), so it's worth one sbrk(0) to confirm.  Wild-high
 * pointers (stack, mmap regions, libasan shadow at 0x511000000000+,
 * kernel addresses) sit far above the ceiling and skip the syscall
 * -- cached_brk_end exists precisely to avoid per-call sbrk(0) on
 * the heap-check hot path.  256 MiB is generous enough to cover any
 * realistic brk grow between refreshes while keeping clearly-not-heap
 * pointers off the syscall path.
 */
#define HEAP_BRK_STALE_SLACK_BYTES (256UL << 20)

static void heap_brk_refresh(void)
{
	unsigned long cur = (unsigned long) sbrk(0);

	if (cur != (unsigned long) -1)
		cached_brk_end = cur;
}

void heap_brk_maybe_refresh(void)
{
	if (++brk_refresh_counter < BRK_REFRESH_INTERVAL)
		return;
	brk_refresh_counter = 0;
	heap_brk_refresh();
}

/*
 * Cached extents of non-brk allocator arenas, captured alongside the
 * [heap] line at heap_bounds_init() time.  glibc's mmap'd arenas, the
 * sanitiser-runtime allocator reservations (libasan primary/secondary
 * at 0x511000000000+, the shadow region, ...), scudo / jemalloc /
 * tcmalloc -- every well-behaved allocator labels its anonymous
 * mappings via prctl(PR_SET_VMA_ANON_NAME), which shows up in
 * /proc/self/maps as "[anon:NAME]" after the inode column.  Trinity
 * itself does not label any of its scratch mappings, so an
 * "[anon:*]" tag in the pre-fork snapshot identifies a region whose
 * contents must not be overwritten by a fuzzed kernel-write
 * argument (the alternative is a write into glibc / libasan chunk
 * metadata, surfacing later as an arena-corruption abort or an
 * ASAN bad-free).
 *
 * 16 entries comfortably covers the regions seen on a libasan-built
 * trinity under glibc 2.39 (one [heap], two glibc mmap arenas, four
 * libasan shadow / allocator regions); an overflow logs once and
 * leaves the trailing regions unprotected -- the wrong direction for
 * safety, so the cap exists to be hit and bumped if a future libsan
 * layout adds more regions.
 */
#define MAX_EXTRA_HEAP_REGIONS	16
struct heap_region {
	unsigned long start;
	unsigned long end;
};
static struct heap_region extra_heap_regions[MAX_EXTRA_HEAP_REGIONS];
static unsigned int nr_extra_heap_regions;

/*
 * Bounding box (min start, max end) over extra_heap_regions[],
 * recomputed atomically in heap_bounds_init() alongside the slot
 * snapshot.  Used by range_overlaps_libc_heap() as a coarse
 * "looks heap-shaped" signal: a query that falls inside the bbox
 * but matches no specific slot is the canonical staleness shape --
 * a post-init secondary mmap (large malloc one-VMA-per-alloc) that
 * landed between the captured slots and is therefore not in the
 * snapshot.  Bbox is empty (end <= start) when no extras have been
 * captured, in which case the predicate degenerates to "never".
 */
static unsigned long extra_heap_regions_bbox_start;
static unsigned long extra_heap_regions_bbox_end;

/*
 * Threshold of "looks heap-shaped but missed all slots" observations
 * before range_overlaps_libc_heap() pays for one /proc/self/maps
 * re-parse and rebuilds extra_heap_regions[].  Set high enough that
 * the common cache-hit path stays at its existing cost (a few
 * compares), low enough that a real post-init secondary mmap is
 * picked up within a small bounded number of misses.  Per-child
 * static -- a child whose own allocator just spawned a new arena
 * refreshes once and then stops missing.
 */
#define HEAP_OUTSIDE_CACHE_REFRESH_THRESHOLD 64

/*
 * Parse a /proc/self/maps line just enough to extract the
 * [start, end), the perms field, and the trailing path/label.  Returns
 * true on success.  The label pointer (which may be NULL or point at
 * the empty string) is written into *label_out; it points into @line,
 * which the caller owns.
 *
 * A line looks like:
 *   55a1b3c00000-55a1b3c21000 rw-p 00000000 00:00 0   [heap]
 *   7f2c1b400000-7f2c1b421000 rw-p 00000000 00:00 0   [anon:libc_malloc]
 *   55a1b3a00000-55a1b3a21000 rw-p 00000000 00:00 0
 * i.e. start-end perms offset major:minor inode optional-label.
 */
static bool parse_maps_line(char *line, unsigned long *start_out,
			    unsigned long *end_out, char perms_out[5],
			    const char **label_out)
{
	unsigned long start, end;
	char perms[8];
	int label_off = -1;
	const char *label;
	char *nl;

	nl = strchr(line, '\n');
	if (nl != NULL)
		*nl = '\0';

	if (sscanf(line, "%lx-%lx %7s %*x %*x:%*x %*u %n",
		   &start, &end, perms, &label_off) < 3)
		return false;
	if (end <= start)
		return false;

	if (label_off < 0 || (size_t) label_off > strlen(line))
		label = "";
	else
		label = line + label_off;

	*start_out = start;
	*end_out = end;
	memcpy(perms_out, perms, 4);
	perms_out[4] = '\0';
	*label_out = label;
	return true;
}

/*
 * Parse /proc/self/maps and stash the brk arena plus every labeled
 * non-brk allocator region.  Called once pre-fork by the parent and
 * once per child from init_child() (after all the post-fork startup
 * mmap traffic has settled) so glibc mmap arenas that the child's
 * own allocator storm spawned after fork are captured -- without the
 * per-child re-parse, those arenas live outside the inherited
 * snapshot and a wild pointer landing in one slips past the overlap
 * gate, letting the kernel scribble glibc chunk metadata and
 * surfacing later as an arena-corruption abort with no proximate
 * reproducer.
 *
 * The parse is committed atomically into module state on success.
 * A failed open (vanishingly rare for /proc/self/maps on our own
 * pid) preserves whatever snapshot was previously in place: the
 * child's COW-inherited parent snapshot stays valid as a fallback
 * rather than collapsing to an empty validator that lets every
 * address through.
 *
 * If the [heap] line is missing (rare: glibc tuned to
 * MALLOC_MMAP_THRESHOLD_=0 or the binary somehow hasn't grown brk
 * yet), heap_start stays 0 and is_in_glibc_heap() falls back to
 * "always true" -- we'd rather permit a marginal free than reject
 * every malloc result on a misconfigured host.
 */
void heap_bounds_init(void)
{
	FILE *f;
	char line[512];
	unsigned long new_heap_start = 0, new_heap_end = 0;
	struct heap_region new_regions[MAX_EXTRA_HEAP_REGIONS];
	unsigned int new_nr = 0;

	f = fopen("/proc/self/maps", "r");
	if (f == NULL) {
		outputerr("heap_bounds_init: open /proc/self/maps failed: %s\n",
			  strerror(errno));
		return;
	}

	while (fgets(line, sizeof(line), f) != NULL) {
		unsigned long start, end;
		char perms[5];
		const char *label;

		if (!parse_maps_line(line, &start, &end, perms, &label))
			continue;

		/*
		 * Only writable private mappings can hold allocator
		 * metadata that the kernel scribbling would corrupt.
		 * Read-only and shared mappings either can't be written
		 * by the kernel (r-- / r-x) or are trinity-controlled
		 * (MAP_SHARED via alloc_shared / track_shared_region,
		 * handled separately by range_overlaps_shared()).
		 */
		if (perms[1] != 'w' || perms[3] != 'p')
			continue;

		if (strncmp(label, "[heap]", 6) == 0 &&
		    (label[6] == '\0' || label[6] == ' ')) {
			new_heap_start = start;
			new_heap_end = end;
			continue;
		}

		/*
		 * "[anon:NAME]" labels come from
		 * prctl(PR_SET_VMA_ANON_NAME) -- glibc malloc tags its
		 * mmap'd arenas, libasan tags its primary / secondary
		 * allocator and shadow reservations, similarly for
		 * scudo / jemalloc / tcmalloc.  Trinity never labels
		 * its own anonymous mappings, so any "[anon:*]" line
		 * in the snapshot belongs to a non-trinity allocator.
		 * Trinity's BSS, stack, vdso, vvar, file-backed
		 * mappings and unlabeled scratch regions are filtered
		 * out by the perms / label tests above.
		 */
		if (strncmp(label, "[anon:", 6) != 0)
			continue;

		if (new_nr >= MAX_EXTRA_HEAP_REGIONS) {
			static bool warned;

			/*
			 * The outputerr fires once per process so the log
			 * doesn't blow up when many regions overflow; the
			 * counter advances on every dropped region so the
			 * post-mortem reader sees the deficit size rather
			 * than just "deficit existed".
			 */
			__atomic_add_fetch(
				&shm->stats.heap_extra_regions_overflow, 1,
				__ATOMIC_RELAXED);

			if (!warned) {
				warned = true;
				outputerr("heap_bounds_init: "
					"MAX_EXTRA_HEAP_REGIONS (%d) reached "
					"-- '%s' and any subsequent allocator "
					"regions are unprotected; raise the "
					"cap\n",
					MAX_EXTRA_HEAP_REGIONS, label);
			}
			continue;
		}

		new_regions[new_nr].start = start;
		new_regions[new_nr].end = end;
		new_nr++;
	}

	fclose(f);

	/*
	 * Commit the freshly-parsed snapshot in one shot.  On the
	 * child-side refresh path this rewrites the COW-inherited
	 * parent snapshot in the child's now-private BSS pages; the
	 * parent's copy and any sibling's copy are unaffected.
	 */
	heap_start = new_heap_start;
	heap_end = new_heap_end;
	memcpy(extra_heap_regions, new_regions,
	       new_nr * sizeof(new_regions[0]));
	nr_extra_heap_regions = new_nr;

	/*
	 * Recompute the extras bbox alongside the slot snapshot.  An
	 * empty snapshot leaves the bbox closed (start=end=0) so the
	 * "looks heap-shaped" predicate in range_overlaps_libc_heap()
	 * cannot fire spuriously when extras are not yet populated.
	 */
	if (new_nr > 0) {
		unsigned long bbox_lo = new_regions[0].start;
		unsigned long bbox_hi = new_regions[0].end;
		unsigned int i;

		for (i = 1; i < new_nr; i++) {
			if (new_regions[i].start < bbox_lo)
				bbox_lo = new_regions[i].start;
			if (new_regions[i].end > bbox_hi)
				bbox_hi = new_regions[i].end;
		}
		extra_heap_regions_bbox_start = bbox_lo;
		extra_heap_regions_bbox_end = bbox_hi;
	} else {
		extra_heap_regions_bbox_start = 0;
		extra_heap_regions_bbox_end = 0;
	}

	/*
	 * Prime the brk cache so the first is_in_glibc_heap() /
	 * range_overlaps_libc_heap() doesn't see 0.
	 */
	heap_brk_refresh();
}

/*
 * Bounds check: is @p inside any captured glibc-managed allocator
 * region?  Accepts the cached brk arena AND the labeled non-brk
 * allocator regions stashed by heap_bounds_init() (glibc mmap arenas,
 * libasan primary/secondary/shadow, scudo / jemalloc / tcmalloc
 * tagged regions -- see extra_heap_regions[]).  Returns true if no
 * allocator extent is known at all (init found neither a [heap] line
 * nor any [anon:NAME] regions) so the caller treats the validator as
 * permissive in that case.
 *
 * Earlier revisions only checked the brk arena, which silently
 * rejected legitimate frees of two important pointer classes:
 *   - allocations above MMAP_THRESHOLD (default 128 KiB), which glibc
 *     services from mmap'd arenas rather than brk, and
 *   - every allocation under ASAN, whose libasan-runtime allocator
 *     hands back chunks from its shadow-mapped pools outside brk.
 * The high deferred_free_reject rate seen in ASAN runs was this gate
 * dropping valid zmalloc() results.
 *
 * Backstop for the bad-free class where a sibling stomp scribbles a
 * snapshot/arg slot with a value that defeats both the pointer-shape
 * heuristic (looks_like_corrupted_ptr) and -- in the worst case --
 * coincidentally matches a tracked malloc result still resident in
 * the alloc-track ring.  An attacker-controlled or wildly-stomped
 * value that lands outside every allocator region (stack, shared
 * region, mmap'd library, executable mapping) is rejected here even
 * if the upstream guards let it through.
 */
bool is_in_glibc_heap(const void *p)
{
	unsigned long v = (unsigned long) p;
	unsigned long end, cur;
	unsigned int i;

	if (heap_start == 0 && nr_extra_heap_regions == 0)
		return true;

	if (heap_start != 0) {
		/*
		 * heap_end is a pre-fork snapshot.  A long-running child
		 * can extend brk past it, so the cached_brk_end snapshot
		 * (refreshed periodically off alloc_object() via
		 * heap_brk_maybe_refresh()) is the live upper bound; brk
		 * only grows in the steady state, so the larger of the two
		 * is the safe outer edge.  cached_brk_end is 0 before its
		 * first refresh and after a sbrk(0) failure -- the max()
		 * falls back to the pre-fork heap_end in that case.
		 */
		end = heap_end;
		cur = cached_brk_end;
		if (cur > end)
			end = cur;

		if (v >= heap_start && v < end)
			return true;

		/*
		 * Mirror of the self-correcting near-edge re-test in
		 * range_overlaps_libc_heap().  The cached snapshot just
		 * judged this pointer not-heap, but brk may have grown past
		 * cached_brk_end since the last heap_brk_maybe_refresh()
		 * tick (heavy non-alloc_object() allocator traffic outruns
		 * the alloc-driven refresh cadence).  Without the re-test, a
		 * deferred-free backstop that lands in [cached_brk_end,
		 * live_brk) marks a real heap chunk as not-heap and silently
		 * drops the free.  Pay one sbrk(0) when the pointer sits in
		 * the staleness slack above the cached bound, refresh the
		 * cache, and re-check; return true if it now falls in the
		 * live arena.  Same slack ceiling
		 * (HEAP_BRK_STALE_SLACK_BYTES) so wild-high pointers skip
		 * the syscall.
		 */
		if (v >= end && v < end + HEAP_BRK_STALE_SLACK_BYTES) {
			unsigned long live_brk = (unsigned long) sbrk(0);

			if (live_brk != (unsigned long) -1) {
				if (live_brk > cached_brk_end)
					cached_brk_end = live_brk;
				if (live_brk > end)
					end = live_brk;
				if (v >= heap_start && v < end) {
					struct childdata *c = this_child();

					if (c != NULL && c->stats_ring != NULL)
						stats_ring_enqueue(c->stats_ring,
								   STATS_FIELD_HEAP_BRK_STALE_WINDOW_HIT,
								   0, 1);
					else
						parent_stats.heap_brk_stale_window_hit++;
					return true;
				}
			}
		}
	}

	for (i = 0; i < nr_extra_heap_regions; i++) {
		if (v >= extra_heap_regions[i].start &&
		    v < extra_heap_regions[i].end)
			return true;
	}

	return false;
}

/*
 * Range-overlap variant of is_in_glibc_heap() with the opposite
 * unknown-bounds polarity: returns true when [addr, addr+len)
 * intersects the cached brk arena or any captured non-brk allocator
 * region (glibc mmap arenas, libasan primary / secondary / shadow,
 * scudo / jemalloc / tcmalloc tagged regions -- see
 * extra_heap_regions[] above).  Used by avoid_shared_buffer() to
 * redirect output-buffer syscall args away from any allocator-managed
 * memory: a fuzzed pointer pointing there lets the kernel scribble
 * chunk metadata, and the next malloc anywhere in trinity finds the
 * corruption and aborts (the cluster from the overnight asan-self-
 * kill triage).  Mirrors range_overlaps_shared() semantics: a single
 * byte of overlap is enough to redirect, and a fully unknown layout
 * (no [heap] line and no captured allocator regions) is treated as
 * no-overlap so we don't redirect every legitimate write.
 */
bool range_overlaps_libc_heap(unsigned long addr, unsigned long len)
{
	unsigned long end, hend, cur;
	unsigned int i;

	/* Treat wrapped ranges as overlapping so callers reject them. */
	if (len != 0 && addr > ULONG_MAX - len)
		return true;

	end = addr + len;
	if (end == addr)
		end = addr + 1;

	if (heap_start != 0) {
		/*
		 * Same brk-grew-past-snapshot story as
		 * is_in_glibc_heap(), with the same cached_brk_end
		 * snapshot for the live upper bound.  Missing the
		 * redirect here is the safety-critical failure mode: a
		 * fuzzed pointer landing in the brk extension above the
		 * cached heap_end gets through avoid_shared_buffer(), the
		 * kernel scribbles glibc chunk metadata in the extension,
		 * and the next malloc in the child aborts.  The cache
		 * staleness window (one BRK_REFRESH_INTERVAL of
		 * alloc_object() calls) is bounded by the refresh cadence
		 * driven off the alloc path -- a brk grow caused by an
		 * allocation gets picked up within INTERVAL allocations.
		 */
		hend = heap_end;
		cur = cached_brk_end;
		if (cur > hend)
			hend = cur;

		if (addr < hend && end > heap_start)
			return true;

		/*
		 * Self-correcting near-edge re-test.  The cached snapshot
		 * just judged this address not-heap, but brk may have grown
		 * past cached_brk_end since the last
		 * heap_brk_maybe_refresh() tick.  Heavy non-alloc_object()
		 * allocator traffic (cmp-hint / RedQueen tables, sequence
		 * record growth) extends the real brk without ticking the
		 * alloc-driven refresh, opening a window where glibc's top
		 * chunk sits in [cached_brk_end, live_brk) and a fuzzed
		 * output pointer in that band gets handed to the kernel
		 * instead of being relocated -- the kernel scribbles
		 * top->size and the next malloc anywhere in the child aborts
		 * with "malloc(): corrupted top size".  Pay one sbrk(0) when
		 * the address sits in the staleness slack above the cached
		 * bound, refresh the cache, and re-test the brk arm; if the
		 * address now falls in the live arena, redirect it (return
		 * true).  Gated by HEAP_BRK_STALE_SLACK_BYTES so wild-high
		 * pointers (kernel addresses, libasan shadow, far mmap) skip
		 * the syscall and the common path stays cached_brk_end-only.
		 * The counter, kept in place, now signals the fix firing
		 * (was: observability-only).
		 */
		if (addr >= hend && addr < hend + HEAP_BRK_STALE_SLACK_BYTES) {
			unsigned long live_brk = (unsigned long) sbrk(0);

			if (live_brk != (unsigned long) -1) {
				if (live_brk > cached_brk_end)
					cached_brk_end = live_brk;
				if (live_brk > hend)
					hend = live_brk;
				if (addr < hend && end > heap_start) {
					struct childdata *c = this_child();

					if (c != NULL && c->stats_ring != NULL)
						stats_ring_enqueue(c->stats_ring,
								   STATS_FIELD_HEAP_BRK_STALE_WINDOW_HIT,
								   0, 1);
					else
						parent_stats.heap_brk_stale_window_hit++;
					return true;
				}
			}
		}
	}

	/*
	 * Walk the captured non-brk allocator regions.  Each entry is
	 * a fixed [start, end) snapshot from heap_bounds_init(), which
	 * the parent runs once pre-fork and each child re-runs at the
	 * end of init_child() so glibc arenas that the child's own
	 * post-fork allocator storm spawned (per-thread mmap arenas,
	 * libasan shadow growth, secondary allocator regions tagged
	 * via PR_SET_VMA_ANON_NAME) make it into the snapshot before
	 * the syscall fuzz loop starts hammering pointers through this
	 * gate.  The captured VMAs are large reservations whose bounds
	 * don't shrink and rarely grow once the child is settled, so a
	 * single refresh per child closes the post-fork window without
	 * touching the hot path.
	 */
	for (i = 0; i < nr_extra_heap_regions; i++) {
		if (addr < extra_heap_regions[i].end &&
		    end > extra_heap_regions[i].start)
			return true;
	}

	/*
	 * Post-init secondary-mmap miss detector.  Falling through to
	 * here means brk and every captured slot rejected the range,
	 * but the query still falls inside the bounding box that spans
	 * the captured slots -- the canonical shape of a libc large-
	 * malloc that bypassed the primary allocator into a fresh
	 * one-VMA-per-alloc landing between two captured arenas after
	 * the heap_bounds_init() snapshot was taken.  Bump an
	 * observability counter so the rate is visible in dump_stats(),
	 * and after every HEAP_OUTSIDE_CACHE_REFRESH_THRESHOLD misses
	 * pay for one /proc/self/maps re-parse and rescan the (now
	 * fresh) extras for this same query.  A genuine post-init mmap
	 * promotes to a real overlap on the rescan and the redirect
	 * fires for the very call that triggered the refresh; a query
	 * inside the bbox that doesn't correspond to any allocator VMA
	 * (sparse extras layout) leaves the snapshot unchanged and the
	 * counter resets, capping the refresh cost at one
	 * /proc/self/maps walk per THRESHOLD misses.
	 */
	if (extra_heap_regions_bbox_end > extra_heap_regions_bbox_start &&
	    addr < extra_heap_regions_bbox_end &&
	    end > extra_heap_regions_bbox_start) {
		static unsigned int outside_cache_since_refresh;
		struct childdata *c;

		c = this_child();
		if (c != NULL && c->stats_ring != NULL)
			stats_ring_enqueue(c->stats_ring,
					   STATS_FIELD_HEAP_POINTER_OUTSIDE_CACHE,
					   0, 1);
		else
			parent_stats.heap_pointer_outside_cache++;

		if (++outside_cache_since_refresh >=
		    HEAP_OUTSIDE_CACHE_REFRESH_THRESHOLD) {
			outside_cache_since_refresh = 0;
			heap_bounds_init();

			for (i = 0; i < nr_extra_heap_regions; i++) {
				if (addr < extra_heap_regions[i].end &&
				    end > extra_heap_regions[i].start)
					return true;
			}
			/* Re-check the brk arena too: heap_bounds_init()
			 * refreshes the brk cache as well, so a query that
			 * tripped because cached_brk_end was stale now
			 * resolves cleanly. */
			if (heap_start != 0) {
				unsigned long hend2 = heap_end;

				if (cached_brk_end > hend2)
					hend2 = cached_brk_end;
				if (addr < hend2 && end > heap_start)
					return true;
			}
		}
	}

	return false;
}

/*
 * Fast-path inverse-polarity check: is [addr, addr+len) fully inside
 * the cached brk arena, or fully inside any single captured non-brk
 * allocator region?  Mirrors range_in_tracked_shared() for the heap
 * snapshot maintained by heap_bounds_init().  Used only by
 * range_readable_user() below -- not a sanitiser gate, so unknown
 * layout returns false (no cached extent implies no proof of
 * readability; the caller treats that as skip-copy).
 */
static bool range_inside_libc_heap(unsigned long addr, unsigned long len)
{
	unsigned long end, hend, cur;
	unsigned int i;

	if (len != 0 && addr > ULONG_MAX - len)
		return false;

	end = addr + len;

	if (heap_start != 0) {
		hend = heap_end;
		cur = cached_brk_end;
		if (cur > hend)
			hend = cur;

		if (addr >= heap_start && end <= hend)
			return true;
	}

	for (i = 0; i < nr_extra_heap_regions; i++) {
		if (addr >= extra_heap_regions[i].start &&
		    end <= extra_heap_regions[i].end)
			return true;
	}

	return false;
}

bool range_readable_user(const void *addr, size_t len)
{
	unsigned long a = (unsigned long) addr;

	if (len == 0)
		return false;
	if (addr == NULL)
		return false;
	if (a > ULONG_MAX - len)
		return false;

	/*
	 * Fast path 1: range is fully inside a tracked shared region.
	 * Trinity owns those mappings outright -- alloc_shared() creates
	 * them PROT_READ|PROT_WRITE and they live for the run, so VMA
	 * presence implies the source bytes are readable.
	 */
	if (range_in_tracked_shared(a, len))
		return true;

	/*
	 * Fast path 2: range is fully inside the cached libc heap (brk
	 * arena) or any captured non-brk allocator region.  Allocator
	 * mappings are PROT_READ|PROT_WRITE by construction; the
	 * heap_bounds_init() snapshot only records writable private VMAs.
	 */
	if (range_inside_libc_heap(a, len))
		return true;

	/*
	 * Unknown layout: a fuzz-introduced VMA outside every cached
	 * snapshot.  Treat as unproven and let the caller route to
	 * asb_relocate()'s no-copy fallback -- chasing the source via a
	 * /proc/self/maps walk on every hot-path call is what this code
	 * was retired to avoid.
	 */
	return false;
}

bool post_snapshot_str(char *dst, size_t dstsz, const char *src)
{
	size_t i;

	if (dst == NULL || dstsz == 0)
		return false;
	if (src == NULL)
		return false;

	/*
	 * Single-probe readability gate.  range_readable_user proves the
	 * full dstsz-byte window of src is mapped (tracked-shared region
	 * or cached libc heap); the copy loop below then never reads past
	 * what we proved.  False here means src is not provably readable
	 * and the caller skips the .post sample rather than feeding a
	 * stale heap-shaped pointer into a downstream strncpy that would
	 * walk off an unrelated allocation.  ASAN catches that walk-off in
	 * test; in production it silently surfaces as an oracle anomaly
	 * against a foreign byte pattern.
	 */
	if (!range_readable_user(src, dstsz))
		return false;

	/*
	 * Same TOCTOU window as post_snapshot_or_skip: a sibling
	 * mprotect/munmap between the readability proof and the read can
	 * fault the src[i] load.  Guard the copy loop with the
	 * asb_copy_active sigsetjmp slot so the fault degrades to a
	 * skipped sample rather than a child crash.
	 */
	if (sigsetjmp(asb_copy_recover, 1) != 0) {
		asb_copy_active = 0;
		return false;
	}
	asb_copy_active = 1;
	for (i = 0; i + 1 < dstsz; i++) {
		char c = src[i];

		dst[i] = c;
		if (c == '\0') {
			asb_copy_active = 0;
			return true;
		}
	}
	dst[i] = '\0';
	asb_copy_active = 0;
	return true;
}

bool post_snapshot_or_skip(void *dst, const void *src, size_t len)
{
	if (src == NULL)
		return false;

	/*
	 * Single-probe readability gate, identical in shape to the one
	 * in post_snapshot_str().  The post oracle's NULL + shape-only
	 * looks_like_corrupted_ptr guard waves through a heap-shaped but
	 * stale/unmapped snap->field; range_readable_user proves the
	 * full len-byte window is mapped (tracked-shared region or
	 * cached libc heap), so the memcpy below cannot fault on the
	 * sibling free / unmap / fuzz-redirect window between the
	 * syscall return and the post sample.  False here means the
	 * caller skips the .post sample rather than feeding the
	 * downstream oracle a foreign byte pattern.
	 */
	if (!range_readable_user(src, len))
		return false;

	/*
	 * range_readable_user() proves src is mapped per trinity's
	 * shared/heap bookkeeping, but a sibling syscall can mprotect or
	 * munmap the tracked region in the window between that check and
	 * this copy.  Guard the memcpy with the asb_copy_active sigsetjmp
	 * slot (the same recovery the get_writable_struct relocate-copy
	 * uses) so a TOCTOU fault skips the .post sample instead of
	 * killing the child.
	 */
	if (sigsetjmp(asb_copy_recover, 1) != 0) {
		asb_copy_active = 0;
		return false;
	}
	asb_copy_active = 1;
	memcpy(dst, src, len);
	asb_copy_active = 0;
	return true;
}

void sanitize_inherited_fds(void)
{
	DIR *dir;
	struct dirent *de;
	int dir_fd;

	dir = opendir("/proc/self/fd");
	if (dir == NULL) {
		outputerr("sanitize_inherited_fds: opendir(/proc/self/fd) failed: %s\n",
			  strerror(errno));
		return;
	}
	dir_fd = dirfd(dir);

	while ((de = readdir(dir)) != NULL) {
		char linkpath[64];
		char target[PATH_MAX];
		char *endp;
		ssize_t n;
		long fdl;
		int fd;

		if (de->d_name[0] == '.')
			continue;

		errno = 0;
		fdl = strtol(de->d_name, &endp, 10);
		if (errno != 0 || *endp != '\0' || fdl < 0 || fdl > INT_MAX)
			continue;
		fd = (int) fdl;

		/* Always keep stdin/stdout/stderr. */
		if (fd <= 2)
			continue;

		/* Skip the readdir() handle itself; closedir() will release
		 * it once the walk completes. */
		if (fd == dir_fd)
			continue;

		n = -1;
		if ((size_t) snprintf(linkpath, sizeof(linkpath),
				      "/proc/self/fd/%d", fd) < sizeof(linkpath))
			n = readlink(linkpath, target, sizeof(target) - 1);
		if (n < 0)
			n = 0;
		target[n] = '\0';

		outputerr("sanitize_inherited_fds: closing unexpected inherited fd %d (%s)\n",
			  fd, n > 0 ? target : "?");

		close(fd);
		if (shm != NULL)
			__atomic_add_fetch(&shm->stats.parent_inherited_fds_closed,
					   1, __ATOMIC_RELAXED);
	}
	closedir(dir);
}

int get_num_fds(void)
{
	struct linux_dirent64 {
		uint64_t       d_ino;
		int64_t        d_off;
		unsigned short d_reclen;
		unsigned char  d_type;
		char           d_name[];
	};
	char path[64];
	char buf[4096];
	int fd, fd_count = 0;
	long nread, pos;

	snprintf(path, sizeof(path), "/proc/%i/fd", mainpid);

	fd = open(path, O_RDONLY | O_DIRECTORY);
	if (fd == -1)
		return 0;

	while ((nread = syscall(SYS_getdents64, fd, buf, sizeof(buf))) > 0) {
		for (pos = 0; pos < nread; ) {
			struct linux_dirent64 *de = (struct linux_dirent64 *)(buf + pos);
			const char *name = de->d_name;

			/* Skip "." and ".." */
			if (!(name[0] == '.' &&
			      (name[1] == '\0' ||
			       (name[1] == '.' && name[2] == '\0'))))
				fd_count++;

			pos += de->d_reclen;
		}
	}

	close(fd);
	return fd_count;
}

/* Plain CRC32 (IEEE 802.3 polynomial, reflected).  Shared by every
 * persistence format trinity emits (minicorpus / cmp_hints /
 * kcov-bitmap) so the headers checksum payloads with one definition
 * instead of byte-identical copies that drift apart silently.  Lazy
 * 256-entry table; first call pays one build, every subsequent call
 * (in any caller) reuses the cached table. */
uint32_t crc32(const void *buf, size_t len)
{
	static uint32_t table[256];
	static bool table_built;
	const uint8_t *p = buf;
	uint32_t crc = 0xffffffffU;
	size_t i;

	if (!table_built) {
		uint32_t c;
		unsigned int n, k;

		for (n = 0; n < 256; n++) {
			c = n;
			for (k = 0; k < 8; k++)
				c = (c & 1) ? (0xedb88320U ^ (c >> 1)) : (c >> 1);
			table[n] = c;
		}
		table_built = true;
	}

	for (i = 0; i < len; i++)
		crc = table[(crc ^ p[i]) & 0xff] ^ (crc >> 8);

	return crc ^ 0xffffffffU;
}