From 11d931059f9865159afcb652c55af983995d53c0 Mon Sep 17 00:00:00 2001 From: raiden00pl Date: Sat, 11 Jul 2026 10:33:17 +0200 Subject: [PATCH] add NuttX Kconfig fuzzing integration (ntfc fuzz) Add a new 'ntfc fuzz' command that fuzzes NuttX Kconfig option subsets and classifies the results Signed-off-by: raiden00pl Assisted-by: Claude Code --- Documentation/fuzzing.rst | 238 ++++++++++ Documentation/index.rst | 1 + config/fuzz/build-stm32.yaml | 28 ++ config/fuzz/mem-sim.yaml | 17 + config/fuzz/ostest-sim.yaml | 27 ++ config/nuttx-sim-nsh.yaml | 13 + pyproject.toml | 3 +- src/ntfc/builder.py | 95 +++- src/ntfc/cli/environment.py | 7 + src/ntfc/cli/main.py | 121 ++++- src/ntfc/commands/cmd_fuzz.py | 74 ++++ src/ntfc/ext_commands.py | 2 + src/ntfc/fuzz/__init__.py | 21 + src/ntfc/fuzz/campaign.py | 162 +++++++ src/ntfc/fuzz/candidate.py | 239 ++++++++++ src/ntfc/fuzz/data/arch-profiles.yaml | 27 ++ src/ntfc/fuzz/data/patterns.yaml | 37 ++ src/ntfc/fuzz/data/scopes.yaml | 29 ++ src/ntfc/fuzz/dataload.py | 106 +++++ src/ntfc/fuzz/discover.py | 178 ++++++++ src/ntfc/fuzz/engine.py | 511 ++++++++++++++++++++++ src/ntfc/fuzz/features.py | 103 +++++ src/ntfc/fuzz/memsize.py | 54 +++ src/ntfc/fuzz/mocks.py | 112 +++++ src/ntfc/fuzz/model.py | 40 ++ src/ntfc/fuzz/profiles.py | 96 ++++ src/ntfc/fuzz/report.py | 145 ++++++ src/ntfc/fuzz/strategy.py | 110 +++++ tests/fuzz/__init__.py | 0 tests/fuzz/conftest.py | 35 ++ tests/fuzz/resources/campaign-build.yaml | 13 + tests/fuzz/resources/campaign-mem.yaml | 10 + tests/fuzz/resources/campaign-ostest.yaml | 13 + tests/fuzz/resources/target-sim.yaml | 10 + tests/fuzz/test_campaign.py | 158 +++++++ tests/fuzz/test_candidate.py | 101 +++++ tests/fuzz/test_cmd_fuzz.py | 65 +++ tests/fuzz/test_dataload.py | 83 ++++ tests/fuzz/test_discover.py | 96 ++++ tests/fuzz/test_engine_build.py | 105 +++++ tests/fuzz/test_engine_helpers.py | 134 ++++++ tests/fuzz/test_engine_mem.py | 103 +++++ tests/fuzz/test_engine_ostest.py | 140 ++++++ tests/fuzz/test_features.py | 88 ++++ tests/fuzz/test_memsize.py | 66 +++ tests/fuzz/test_mocks.py | 80 ++++ tests/fuzz/test_model.py | 30 ++ tests/fuzz/test_packaged_data.py | 28 ++ tests/fuzz/test_profiles.py | 80 ++++ tests/fuzz/test_report_write.py | 87 ++++ tests/fuzz/test_strategy.py | 85 ++++ tests/test_builder.py | 151 +++++++ 52 files changed, 4354 insertions(+), 3 deletions(-) create mode 100644 Documentation/fuzzing.rst create mode 100644 config/fuzz/build-stm32.yaml create mode 100644 config/fuzz/mem-sim.yaml create mode 100644 config/fuzz/ostest-sim.yaml create mode 100644 config/nuttx-sim-nsh.yaml create mode 100644 src/ntfc/commands/cmd_fuzz.py create mode 100644 src/ntfc/fuzz/__init__.py create mode 100644 src/ntfc/fuzz/campaign.py create mode 100644 src/ntfc/fuzz/candidate.py create mode 100644 src/ntfc/fuzz/data/arch-profiles.yaml create mode 100644 src/ntfc/fuzz/data/patterns.yaml create mode 100644 src/ntfc/fuzz/data/scopes.yaml create mode 100644 src/ntfc/fuzz/dataload.py create mode 100644 src/ntfc/fuzz/discover.py create mode 100644 src/ntfc/fuzz/engine.py create mode 100644 src/ntfc/fuzz/features.py create mode 100644 src/ntfc/fuzz/memsize.py create mode 100644 src/ntfc/fuzz/mocks.py create mode 100644 src/ntfc/fuzz/model.py create mode 100644 src/ntfc/fuzz/profiles.py create mode 100644 src/ntfc/fuzz/report.py create mode 100644 src/ntfc/fuzz/strategy.py create mode 100644 tests/fuzz/__init__.py create mode 100644 tests/fuzz/conftest.py create mode 100644 tests/fuzz/resources/campaign-build.yaml create mode 100644 tests/fuzz/resources/campaign-mem.yaml create mode 100644 tests/fuzz/resources/campaign-ostest.yaml create mode 100644 tests/fuzz/resources/target-sim.yaml create mode 100644 tests/fuzz/test_campaign.py create mode 100644 tests/fuzz/test_candidate.py create mode 100644 tests/fuzz/test_cmd_fuzz.py create mode 100644 tests/fuzz/test_dataload.py create mode 100644 tests/fuzz/test_discover.py create mode 100644 tests/fuzz/test_engine_build.py create mode 100644 tests/fuzz/test_engine_helpers.py create mode 100644 tests/fuzz/test_engine_mem.py create mode 100644 tests/fuzz/test_engine_ostest.py create mode 100644 tests/fuzz/test_features.py create mode 100644 tests/fuzz/test_memsize.py create mode 100644 tests/fuzz/test_mocks.py create mode 100644 tests/fuzz/test_model.py create mode 100644 tests/fuzz/test_packaged_data.py create mode 100644 tests/fuzz/test_profiles.py create mode 100644 tests/fuzz/test_report_write.py create mode 100644 tests/fuzz/test_strategy.py diff --git a/Documentation/fuzzing.rst b/Documentation/fuzzing.rst new file mode 100644 index 0000000..a805cf4 --- /dev/null +++ b/Documentation/fuzzing.rst @@ -0,0 +1,238 @@ +==================== +Kconfig Fuzzing +==================== + +NTFC can fuzz NuttX Kconfig options on top of a known-good base configuration +to surface three classes of problem: + +* **Build breaks** -- an option that no longer compiles (for example a now + shared ``CONFIG_STM32_*`` option enabled on a family whose driver was never + ported). +* **Broken features at runtime** -- an option (or combination) that builds but + makes ``ostest`` crash, hang, or fail. +* **Memory cost** -- the flash/RAM footprint an option or feature adds. + +The fuzzer generates many candidate configurations from one base config and +drives each through NTFC's existing build (:class:`~ntfc.builder.NuttXBuilder`) +and device (sim / QEMU / serial) layers. A candidate is simply the base +``defconfig`` plus a set of Kconfig overrides -- the same ``kv`` mechanism NTFC +already uses -- so no transient board directory is created. + +Two kinds of config +=================== + +The fuzzer separates *what to fuzz* from *where to run it*: + +* The **fuzz config** is fuzzer-only: feature, surface, strategy, mock, + workers/jobs. It has no device/board/flash concepts. +* The **target** is a normal NTFC config (device, board defconfig, QEMU + ``exec_args``, serial port, ``flash``/``reboot``). It is reused as-is for + sim, QEMU, or real hardware. + +How many configs a run needs depends on whether it boots a device: + +.. list-table:: + :header-rows: 1 + + * - Feature + - Boots a device? + - Configs + * - ``build`` + - no -- only compiles + - **1** -- the fuzz config (it names ``board`` + ``tree``) + * - ``mem`` + - no -- compiles + sizes ELF + - **1** -- the fuzz config + * - ``ostest`` + - yes -- runs the target + - **2** -- fuzz config **+** NTFC target (``--confpath``) + +Usage +===== + +.. code-block:: bash + + # only discover and print the fuzz surface (no build): + ntfc fuzz --campaign config/fuzz/build-stm32.yaml --list + + # print the planned candidate matrix (no build; for ostest this works + # without --confpath -- the matrix depends only on the fuzz config): + ntfc fuzz --campaign config/fuzz/build-stm32.yaml --dry-run + + # build-break sweep (one config -- board + tree are in the fuzz config): + ntfc fuzz --campaign config/fuzz/build-stm32.yaml + + # memory footprint report (one config): + ntfc fuzz --campaign config/fuzz/mem-sim.yaml + + # ostest sweep (TWO configs: fuzz config + NTFC target): + ntfc fuzz --campaign config/fuzz/ostest-sim.yaml \ + --confpath config/nuttx-sim-nsh.yaml + +The report (``fuzz-report.txt`` and ``fuzz-report.json``) is written to the +NTFC session directory, beside normal test results, and the full build/run log +of every non-pass candidate is saved next to the build. The exit code is +non-zero if any candidate did not pass. + +Requirements +============ + +* ``kconfiglib`` (installed as an NTFC dependency) -- Kconfig parsing. +* ``cmake`` and ``ninja`` -- the build backend, as for any NTFC build. +* A ``tree`` dir containing sibling ``nuttx/`` and ``apps/`` checkouts. + +Fuzz config +=========== + +A fuzz config is a flat, fuzzer-only YAML file. + +.. code-block:: yaml + + # fuzz-build.yaml -- a build-break sweep (build/mem carry their own target) + feature: build # build | ostest | mem + arch: stm32 # arch profile (skip/mock/board-required rules) + + board: nucleo-h563zi:nsh # build/mem only: board:config to build + tree: ./external # build/mem only: dir with nuttx/ and apps/ + + surface: # WHAT to fuzz -- choose one style: + scope: [arch] # by subsystem (see "Scopes" below) + # symbols: [NET_TCP, FS_FAT] # or explicit symbol names + # features: # or named feature groups + # net: [CONFIG_NET, CONFIG_NETDEV_LATEINIT] + include_choices: false + + strategy: # HOW to combine + mode: single # single | random | pairs | marginal | full + limit: 10 # cap the candidate count (0/absent = no cap) + rounds: 20 # random mode + size: 4 # random mode + seed: 0 # random mode + minimize: true # delta-debug failing combinations + + mock: true # build-validation mock mode + require: [CONFIG_SCHED_HPWORK] # options enabled in every build + timeout: 600 # ostest per-run seconds + + parallel: true # fuzzer orchestration + workers: 4 # candidates built/run concurrently + jobs: 4 # ninja jobs per build + +For ``ostest`` the ``board`` and ``tree`` keys are omitted -- the target +(board, device, run parameters) comes entirely from the NTFC config passed with +``--confpath``. + +``feature`` +----------- + +Which of the three capabilities to run: + +* ``build`` -- build each candidate; report ``pass`` / ``build-fail`` (and + ``pass-mocked`` when ``mock`` is set). +* ``ostest`` -- build and run each candidate on the ``--confpath`` target's + device, classifying ``pass`` / ``test-fail`` / ``crash`` / ``timeout`` and + delta-debugging failing combinations to a minimal set. +* ``mem`` -- build the base and per-feature variants and diff the linked image + sizes into a flash/RAM cost table. + +``surface`` +----------- + +Selects the fuzz surface, in one of three styles: + +* ``scope`` -- a list of subsystem names (see `Scopes`_). +* ``symbols`` -- explicit Kconfig symbol names (without the ``CONFIG_`` + prefix). +* ``features`` -- named groups of options treated as single units + (``mem`` / ``ostest``). + +``include_choices`` (default ``false``) also fuzzes members of ``choice`` +blocks. + +``strategy`` +------------ + +* ``mode`` -- ``single`` (one option at a time), ``random`` (random subsets), + or the systematic modes ``marginal`` / ``pairs`` / ``full``. +* ``limit`` -- cap the number of candidates (the first N of the sweep); use it + to keep a large discovered surface bounded. Preview the matrix with + ``--dry-run``. ``mem`` uses ``max_builds`` (default 64) instead. +* ``rounds`` / ``size`` / ``seed`` -- random-mode controls (seed makes a run + reproducible). +* ``minimize`` (default ``true``) -- delta-debug each failing combination to + the smallest subset that still fails. + +Data files +========== + +Everything the fuzzer "knows" that changes over time lives in editable YAML +data files under ``src/ntfc/fuzz/data/`` (shipped as package data, +overridable). Maintaining the fuzz surface never requires editing code. + +``arch-profiles.yaml`` + Per-architecture rules: which Kconfig files form the arch surface + (``symbol_path``), which symbol prefixes are never toggled + (``skip_prefixes``), which build-error identifiers are board-supplied and + therefore mockable (``mockable_prefixes`` / ``mockable_suffixes``), and any + arch-specific board-config-requirement patterns (``extra_board_required``). + Add a new architecture by adding an entry -- no code change. + +``scopes.yaml`` + The subsystem vocabulary: a name (``net``, ``fs``, ``usb``, ``drivers``, + ``kernel`` ...) mapped to a regex matched against a symbol's defining + Kconfig file path. Add a subsystem with one line. + +``patterns.yaml`` + The classification regex tables: ``board_required`` (unmet board + prerequisites, not code bugs), ``config_required`` (Kconfig dependency + problems), ``build_error`` (log excerpting), and ``ostest_exit`` / ``crash`` + (runtime classification, complementing the device layer's own crash + detection). + +.. _scopes: + +Scopes +------ + +``scope`` accepts ``arch`` (the arch profile's own surface), ``all`` (the whole +tree), or any named subsystem from ``scopes.yaml``, comma-separated -- for +example ``[net, fs, audio]``. Discovery only returns options that are actually +settable from the chosen base, so to fuzz a subsystem deeply pick a base config +that already enables its root. Beware of subsystems gated behind a single +``menuconfig``: on a base config that does not enable it, a scope like +``[can]`` collapses to just that one root option (everything below it is not +yet settable) and the sweep degenerates to a single candidate. Check the +surface first with ``--list``; ``arch`` on a chip base config is usually the +richest build-break surface. + +Mock mode +========= + +Many peripherals fail to build only because the board's ``board.h`` does not +define the pin mapping or geometry constants for a peripheral it never wires +(``'GPIO_CAN2_RX' undeclared``, ``#error BOARD_LTDC_WIDTH must be defined``). +With ``mock: true`` the fuzzer reads those errors, synthesises just the missing +*board-supplied* constants (using gcc's own "did you mean" suggestion for pin +alternatives), injects them into the build's generated ``config.h`` -- touching +no source -- and rebuilds. Only an allowlist of board-constant prefixes is +mocked, so genuine code bugs still fail the build. Results split into +``pass-mocked`` (the arch path compiles once board pins are provided) and +``build-fail`` (a real code bug). + +Mocked builds validate **compilation**, not runtime correctness. + +Caveats +======= + +* The ostest sweep boots the base config to NSH, runs the ``ostest`` builtin + over the device layer, and treats the ``ostest_main: Exiting with status N`` + line as authoritative (0 -> pass, else test-fail). The **simulator is the + natural first target** and works well; note the full ostest suite is not + real-time under the sim and runs the whole suite each candidate, so a single + run takes a few minutes (≈3 min on the ``sim:nsh`` baseline) -- set + ``timeout`` accordingly and raise ``workers`` to parallelise. QEMU and real + serial targets work the same way via ``device: qemu`` / ``serial``. The full + console output of every non-pass run is saved next to the build for + inspection. +* ``mem`` reports the static footprint of the linked image; it does not model + runtime heap or stack. diff --git a/Documentation/index.rst b/Documentation/index.rst index 5383f43..b46a6ba 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -14,6 +14,7 @@ for NuttX. writing-test-cases config-yaml multi-session + fuzzing session-json report api diff --git a/config/fuzz/build-stm32.yaml b/config/fuzz/build-stm32.yaml new file mode 100644 index 0000000..f32ad3c --- /dev/null +++ b/config/fuzz/build-stm32.yaml @@ -0,0 +1,28 @@ +# Build-break sweep: toggle each STM32 peripheral option of the chip (scope +# 'arch') on top of the base config and try to compile it. 'mock: true' +# synthesizes missing board constants (GPIO_*, DMAMAP_*, ...) so peripheral +# drivers compile without board glue, separating real build breaks from +# missing-board-config noise. +# +# build only compiles, so ONE config (this fuzz config) is enough -- it names +# the board and the tree to build. +# +# ntfc fuzz --campaign config/fuzz/build-stm32.yaml --list # surface +# ntfc fuzz --campaign config/fuzz/build-stm32.yaml --dry-run # matrix +# ntfc fuzz --campaign config/fuzz/build-stm32.yaml # sweep +# +# 'limit' keeps the demo bounded (first N of the sweep); drop it to sweep the +# whole surface (~70 candidates on this board). +feature: build +arch: stm32 +board: nucleo-h563zi:nsh # board:config to fuzz +tree: ./external # dir containing nuttx/ and apps/ +surface: + scope: [arch] +strategy: + mode: single # one candidate per option + limit: 10 +mock: true +parallel: true +workers: 4 +jobs: 4 diff --git a/config/fuzz/mem-sim.yaml b/config/fuzz/mem-sim.yaml new file mode 100644 index 0000000..21152a1 --- /dev/null +++ b/config/fuzz/mem-sim.yaml @@ -0,0 +1,17 @@ +# Memory footprint report: build the baseline and each variant, then diff the +# flash/RAM footprint of every option against the baseline. All three symbols +# are off in sim:nsh, so each one produces a real, non-zero delta. +# +# mem only compiles + sizes the ELF, so ONE config is enough. +# +# ntfc fuzz --campaign config/fuzz/mem-sim.yaml --dry-run # matrix +# ntfc fuzz --campaign config/fuzz/mem-sim.yaml # sweep +feature: mem +arch: sim +board: sim:nsh +tree: ./external +surface: + symbols: [CRYPTO, FS_TMPFS, LIBC_FLOATINGPOINT] +strategy: + mode: marginal # each symbol alone + all together +jobs: 4 diff --git a/config/fuzz/ostest-sim.yaml b/config/fuzz/ostest-sim.yaml new file mode 100644 index 0000000..e8c73d7 --- /dev/null +++ b/config/fuzz/ostest-sim.yaml @@ -0,0 +1,27 @@ +# ostest sweep: build random combinations of the listed options, boot each +# one, run ostest over the device layer, and delta-debug any runtime failure +# down to a minimal failing option set. All three symbols are off in sim:nsh, +# so every candidate exercises a genuinely different kernel configuration. +# +# ostest boots a real target, so it needs TWO configs: +# 1. this fuzz config (what to fuzz) +# 2. an NTFC target config passed with --confpath (device/board/run params) +# +# ntfc fuzz --campaign config/fuzz/ostest-sim.yaml --dry-run # matrix +# ntfc fuzz --campaign config/fuzz/ostest-sim.yaml \ +# --confpath config/nuttx-sim-nsh.yaml # sweep +# +# The target is a normal NTFC config, so swap it for a QEMU or serial target +# (device: qemu / serial) to fuzz ostest on emulation or real hardware. +feature: ostest +arch: sim +surface: + symbols: [SCHED_LPWORK, PRIORITY_INHERITANCE, MM_KERNEL_HEAP] +strategy: + mode: random + rounds: 4 + size: 3 + seed: 8 # this seed yields 4 distinct subsets + minimize: true +timeout: 600 +jobs: 4 diff --git a/config/nuttx-sim-nsh.yaml b/config/nuttx-sim-nsh.yaml new file mode 100644 index 0000000..317333e --- /dev/null +++ b/config/nuttx-sim-nsh.yaml @@ -0,0 +1,13 @@ +# Simulator target booting the nsh defconfig (with the ostest builtin). +# 'cwd' must contain nuttx/ and apps/. Used e.g. as the ostest fuzzing target: +# ntfc fuzz --campaign config/fuzz/ostest-sim.yaml --confpath +config: + cwd: './external' + build_dir: './build' +product: + name: "sim-nsh" + cores: + core0: + name: 'main' + device: 'sim' + defconfig: 'boards/sim/sim/sim/configs/nsh' diff --git a/pyproject.toml b/pyproject.toml index 58ceba7..71c84f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ requires-python = ">=3.10" dependencies = [ "click>=8.1", "pyyaml", + "kconfiglib", # Kconfig parsing for the fuzz feature "pyelftools", "pexpect", "psutil", @@ -57,7 +58,7 @@ keywords = [ ntfc = "ntfc.cli.main:main" [tool.setuptools.package-data] -ntfc = ["log.yaml", "log/templates/*.html"] +ntfc = ["log.yaml", "log/templates/*.html", "fuzz/data/*.yaml"] [tool.black] line-length = 79 diff --git a/src/ntfc/builder.py b/src/ntfc/builder.py index ace5eeb..6636d7a 100644 --- a/src/ntfc/builder.py +++ b/src/ntfc/builder.py @@ -24,8 +24,9 @@ import re import shutil import subprocess +from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from ntfc.log.logger import logger @@ -34,6 +35,16 @@ class BuilderConfigError(ValueError): """Invalid build configuration in YAML.""" +@dataclass +class CandidateBuild: + """Result of an ephemeral fuzz-candidate build.""" + + ok: bool + log: str + elf_path: str + conf_path: str + + class NuttXBuilder: """NuttX configuration builder (CMake only).""" @@ -266,6 +277,12 @@ def _apply_kconfig_overrides( if key not in replaced: updated_lines.append(self._format_kconfig_line(key, value)) + # Skip identical rewrites: touching .config forces the build to + # regenerate config.h, which discards constants the fuzzer's mock + # mode injected there between build retries. + if updated_lines == lines: + return + with open(conf_path, "w", encoding="utf-8") as f: f.writelines(updated_lines) @@ -445,6 +462,82 @@ def _flash_core( logger.info(f"flash image cmd: {cmd}") self._run_command(cmd, env=None) + def _run_capture( + self, cmd: List[str], env: Optional[Dict[str, str]] = None + ) -> Tuple[int, str]: # pragma: no cover + """Run a command, capturing output; never raises on non-zero.""" + run_env = os.environ.copy() + if env: + run_env.update(env) + proc = subprocess.run(cmd, env=run_env, text=True, capture_output=True) + return proc.returncode, proc.stdout + proc.stderr + + def build_candidate( + self, + board_config: str, + kv: Dict[str, Any], + build_dir: str, + jobs: Optional[int] = None, + build_env: Optional[Dict[str, str]] = None, + configure: bool = True, + ) -> "CandidateBuild": + """Build one fuzz candidate into a unique dir, capturing failures. + + A candidate is the base board's defconfig plus the ``kv`` Kconfig + overrides (the fuzz toggles), applied to the generated ``.config`` + before the build. Nothing in the source tree is modified. Unlike + :meth:`build_all`, a build failure is captured and returned rather than + raised, so a sweep can classify it. + + ``configure=False`` compiles the existing build dir as-is (no cmake + configure, no override re-apply, no config re-expansion). The fuzzer's + mock retries need this: reconfiguring regenerates ``config.h`` and + would discard the mock constants injected there. + """ + cfg_cwd = self._cfg_values["config"]["cwd"] + nuttx_dir = os.path.join(cfg_cwd, "nuttx") + elf_path = os.path.join(build_dir, "nuttx") + conf_path = os.path.join(build_dir, ".config") + log = "" + + if configure: + self._make_dir(Path(build_dir)) + cfg_cmd = [ + "cmake", + f"-B{build_dir}", + f"-S{nuttx_dir}", + "-GNinja", + f"-DBOARD_CONFIG={board_config}", + ] + rc, log = self._run_capture(cfg_cmd, build_env) + if rc != 0: + return CandidateBuild(False, log, elf_path, conf_path) + + # reuse the existing .config override machinery (no source + # touched); called without cfg_cwd so it edits .config directly + # (no tweak tool) + self._apply_kconfig_overrides(conf_path, kv) + + if kv: + # Re-expand the edited .config (as 'make olddefconfig'): + # options unlocked by the overrides need their defaults + # materialized, or dependent int/hex symbols are missing from + # config.h and the enabled code fails with 'undeclared' + # errors. + rc, olog = self._run_capture( + ["cmake", "--build", build_dir, "-t", "olddefconfig"], + build_env, + ) + log += olog + if rc != 0: + return CandidateBuild(False, log, elf_path, conf_path) + + build_cmd = ["cmake", "--build", build_dir] + if jobs: + build_cmd += ["--", f"-j{jobs}"] + rc, blog = self._run_capture(build_cmd, build_env) + return CandidateBuild(rc == 0, log + blog, elf_path, conf_path) + def need_build(self) -> bool: """Check if we need build something.""" for product in self._cfg_values: diff --git a/src/ntfc/cli/environment.py b/src/ntfc/cli/environment.py index 891f3e4..54bb0d5 100644 --- a/src/ntfc/cli/environment.py +++ b/src/ntfc/cli/environment.py @@ -62,6 +62,13 @@ class DEnvironmentData: runmulti: bool = False manifest: Optional[str] = None + # fuzzing + runfuzz: bool = False + fuzzpath: Optional[str] = None + fuzz_confpath: Optional[str] = None + fuzz_list: bool = False + fuzz_dry_run: bool = False + # files testpath: Optional[str] = None confpath: Optional[str] = None diff --git a/src/ntfc/cli/main.py b/src/ntfc/cli/main.py index 9fde77a..31afd19 100644 --- a/src/ntfc/cli/main.py +++ b/src/ntfc/cli/main.py @@ -340,8 +340,120 @@ def multi_run(ctx: Environment) -> int: return runner.run() +def _fuzz_check_tree(cwd: Any, where: str) -> bool: # pragma: no cover + """Verify a tree dir contains nuttx/ and apps/; print guidance if not.""" + from pathlib import Path + + cwd = Path(cwd) + if (cwd / "nuttx").is_dir() and (cwd / "apps").is_dir(): + return True + print( + f"[fuzz] ERROR: {where} '{cwd}' must contain both nuttx/ and apps/ " + f"checkouts (nuttx/ present: {(cwd / 'nuttx').is_dir()}, apps/ " + f"present: {(cwd / 'apps').is_dir()}). Nothing was built." + ) + return False + + +def fuzz_run(ctx: Environment) -> int: # pragma: no cover # noqa: C901 + """Run a fuzz config and write its report into a session directory. + + ``build`` / ``mem`` need only the fuzz config (it names board + tree). + ``ostest`` also needs an NTFC target config via ``--confpath``. + """ + from pathlib import Path + + from ntfc.fuzz import dataload, discover + from ntfc.fuzz.campaign import ( + CampaignError, + board_config_of, + load_fuzz_config, + ) + from ntfc.fuzz.engine import ( + _label, + _surface_names, + plan_candidates, + run_campaign, + ) + from ntfc.log.manager import LogManager + + assert ctx.fuzzpath is not None + try: + fuzz = load_fuzz_config(ctx.fuzzpath) + except CampaignError as exc: + print(f"[fuzz] ERROR: {exc}") + return 2 + + # The ostest candidate matrix comes from the fuzz config alone, so a + # --dry-run does not need the target config. + target = None + cwd = None + board = "(target from --confpath)" + nuttx_root = Path(".") + if fuzz.needs_target: + if ctx.fuzz_confpath: + with open(ctx.fuzz_confpath, "r", encoding="utf-8") as f: + target = yaml.safe_load(f) + cwd = Path(target["config"]["cwd"]).resolve() + target["config"]["cwd"] = str(cwd) + board = board_config_of(target) + nuttx_root = cwd / "nuttx" + elif not ctx.fuzz_dry_run: + print( + "[fuzz] ERROR: the 'ostest' feature needs a target config: " + "pass --confpath (device/board)." + ) + return 2 + else: + cwd = Path(str(fuzz.tree)).resolve() + fuzz.tree = str(cwd) + fuzz.build_dir = str(Path(fuzz.build_dir).resolve()) + board = str(fuzz.board) + nuttx_root = cwd / "nuttx" + + if cwd is not None and not _fuzz_check_tree(cwd, "the tree"): + return 2 + + if ctx.fuzz_list: + profile = dataload.get_profile(fuzz.arch) + names = _surface_names( + fuzz, nuttx_root, profile, board, discover.discover + ) + print(f"fuzz surface for {board}:") + for n in names: + print(f" {n}") + return 0 + + if ctx.fuzz_dry_run: + subsets = plan_candidates(fuzz, target) + print( + f"[fuzz] planned {fuzz.feature} candidates for {board}: " + f"{len(subsets)}" + ) + for s in subsets: + print(f" {_label(s)}") + return 0 + + def _progress(msg: str) -> None: + print(msg, flush=True) + + print( + f"[fuzz] starting {fuzz.feature} campaign on {board} ...", flush=True + ) + report = run_campaign(fuzz, target, progress=_progress) + + log_manager = LogManager(ctx.result.get("logcfg") if ctx.result else None) + log_manager.cleanup() + session_dir = log_manager.new_session_dir() + report.write(session_dir) + + print(report.text()) + print(f"\n[fuzz] report written to {session_dir}") + return 1 if report.failed() else 0 + + @pass_environment -def cli_on_close(ctx: Environment) -> bool: +def cli_on_close(ctx: Environment) -> bool: # noqa: C901 """Handle all work on Click close.""" if ctx.helpnow: # pragma: no cover # do nothing if help was called @@ -354,6 +466,13 @@ def cli_on_close(ctx: Environment) -> bool: exit(1) return True + # fuzzing mode + if ctx.runfuzz: + ret = fuzz_run(ctx) + if ret != 0: + exit(1) + return True + # load configuration try: conf, conf_json = load_config_files(ctx) diff --git a/src/ntfc/commands/cmd_fuzz.py b/src/ntfc/commands/cmd_fuzz.py new file mode 100644 index 0000000..2ba0cf0 --- /dev/null +++ b/src/ntfc/commands/cmd_fuzz.py @@ -0,0 +1,74 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Module containing NTFC fuzz command.""" + +from typing import Optional + +import click + +from ntfc.cli.environment import Environment, pass_environment + +############################################################################### +# Command: cmd_fuzz +############################################################################### + + +@click.command(name="fuzz") +@click.option( + "--campaign", + required=True, + type=click.Path(resolve_path=False), + help="Path to a fuzz config YAML file (what to fuzz).", +) +@click.option( + "--confpath", + default=None, + type=click.Path(resolve_path=False), + help="NTFC target config (device/board). Required for the ostest feature; " + "ignored for build/mem, which build the fuzz config's own board/tree.", +) +@click.option( + "--list", + "list_surface", + is_flag=True, + help="Only discover and print the fuzz surface; do not build.", +) +@click.option( + "--dry-run", + is_flag=True, + help="Print the planned candidate matrix without building.", +) +@pass_environment +def cmd_fuzz( + ctx: Environment, + campaign: str, + confpath: Optional[str], + list_surface: bool, + dry_run: bool, +) -> bool: + """Run a Kconfig fuzzing campaign (build / ostest / mem).""" + ctx.runfuzz = True + ctx.fuzzpath = campaign + ctx.fuzz_confpath = confpath + ctx.fuzz_list = list_surface + ctx.fuzz_dry_run = dry_run + + return True diff --git a/src/ntfc/ext_commands.py b/src/ntfc/ext_commands.py index 89db75e..aac93ae 100644 --- a/src/ntfc/ext_commands.py +++ b/src/ntfc/ext_commands.py @@ -24,6 +24,7 @@ from ntfc.commands.cmd_build import cmd_build from ntfc.commands.cmd_collect import cmd_collect +from ntfc.commands.cmd_fuzz import cmd_fuzz from ntfc.commands.cmd_test import cmd_test if TYPE_CHECKING: @@ -33,4 +34,5 @@ cmd_build, cmd_collect, cmd_test, + cmd_fuzz, ] diff --git a/src/ntfc/fuzz/__init__.py b/src/ntfc/fuzz/__init__.py new file mode 100644 index 0000000..2961725 --- /dev/null +++ b/src/ntfc/fuzz/__init__.py @@ -0,0 +1,21 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""NuttX Kconfig fuzzing integrated into NTFC.""" diff --git a/src/ntfc/fuzz/campaign.py b/src/ntfc/fuzz/campaign.py new file mode 100644 index 0000000..b117513 --- /dev/null +++ b/src/ntfc/fuzz/campaign.py @@ -0,0 +1,162 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Fuzz configuration parsing. + +A *fuzz config* is a fuzzer-only YAML file: what to fuzz (feature, surface), +how (strategy, mock), and fuzzer orchestration (workers, jobs). It carries no +NTFC/device concepts. + +The build/run *target* is separate: + +* ``build`` and ``mem`` only compile, so the fuzz config itself names the build + target (``board`` + ``tree``) -- a single config is enough. +* ``ostest`` boots a real target, so the target is a normal NTFC config passed + with ``--confpath`` (sim / QEMU / serial, with all its device/flash options). + Two configs are used: this fuzz config plus the NTFC target config. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import yaml # type: ignore + +FEATURES = ("build", "ostest", "mem") +_SURFACE_KEYS = ("scope", "symbols", "features") +# Features that only compile and therefore carry their own build target. +_BUILD_ONLY = ("build", "mem") + + +class CampaignError(ValueError): + """Invalid fuzz configuration.""" + + +@dataclass +class FuzzConfig: + """A parsed fuzzer-only configuration.""" + + feature: str + surface: Dict[str, Any] + strategy: Dict[str, Any] + arch: str = "stm32" + mock: bool = False + require: List[str] = field(default_factory=list) + timeout: float = 90.0 + workers: int = 1 + jobs: Optional[int] = None + parallel: bool = False + # Build target for build/mem (unused for ostest -- that target is the + # NTFC config passed via --confpath): + board: Optional[str] = None + tree: Optional[str] = None + build_dir: str = "./build" + + @property + def needs_target(self) -> bool: + """Whether this feature needs an external NTFC target config.""" + return self.feature not in _BUILD_ONLY + + +def _load_yaml(path: str) -> Dict[str, Any]: + try: + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + except FileNotFoundError as exc: + raise CampaignError(f"file not found: {path}") from exc + if not isinstance(data, dict): + raise CampaignError(f"not a YAML mapping: {path}") + return data + + +def board_config_of(target_conf: Dict[str, Any]) -> str: + """Derive the ``board:config`` string from an NTFC config's defconfig. + + A defconfig path ``boards////configs/`` yields + ``:``. + """ + for key, val in target_conf.items(): + if "product" not in key or not isinstance(val, dict): + continue + for core in val.get("cores", {}).values(): + defconfig = core.get("defconfig") + if not defconfig: + continue + parts = defconfig.strip("/").split("/") + if "configs" in parts: + idx = parts.index("configs") + return f"{parts[idx - 1]}:{parts[idx + 1]}" + raise CampaignError("no core defconfig found in target config") + + +def build_target_conf(tree: str, build_dir: str) -> Dict[str, Any]: + """Minimal NuttXBuilder config for a build-only target (build/mem). + + NuttXBuilder only needs ``config.cwd`` (the dir containing ``nuttx/`` and + ``apps/``) and ``config.build_dir``; the board is passed to it directly. + """ + return {"config": {"cwd": tree, "build_dir": build_dir}} + + +def load_fuzz_config(path: str) -> FuzzConfig: + """Load and validate a fuzzer-only config file.""" + doc = _load_yaml(path) + + feature = doc.get("feature") + if feature not in FEATURES: + raise CampaignError( + f"'feature' must be one of {FEATURES}, got {feature!r}" + ) + + surface = doc.get("surface") or {} + if not any(surface.get(k) for k in _SURFACE_KEYS): + raise CampaignError( + "'surface' must set one of scope / symbols / features" + ) + + strategy = dict(doc.get("strategy") or {}) + strategy.setdefault("mode", "single") + + board = doc.get("board") + tree = doc.get("tree") + if feature in _BUILD_ONLY: + if not board: + raise CampaignError( + f"'{feature}' needs a 'board' (board:config) to build" + ) + if not tree: + raise CampaignError( + f"'{feature}' needs a 'tree' (dir with nuttx/ and apps/)" + ) + + return FuzzConfig( + feature=feature, + surface=surface, + strategy=strategy, + arch=doc.get("arch", "stm32"), + mock=bool(doc.get("mock", False)), + require=list(doc.get("require") or []), + timeout=float(doc.get("timeout", 90.0)), + workers=int(doc.get("workers", 1)), + jobs=doc.get("jobs"), + parallel=bool(doc.get("parallel", False)), + board=board, + tree=tree, + build_dir=doc.get("build_dir", "./build"), + ) diff --git a/src/ntfc/fuzz/candidate.py b/src/ntfc/fuzz/candidate.py new file mode 100644 index 0000000..fe4d1df --- /dev/null +++ b/src/ntfc/fuzz/candidate.py @@ -0,0 +1,239 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Build, run, and classify one fuzz candidate. + +A candidate is a base config plus a set of ``CONFIG_*`` toggles. The build path +goes through :meth:`ntfc.builder.NuttXBuilder.build_candidate`; the run path +boots the built image through NTFC's device layer (sim / QEMU / serial) and +classifies the outcome. ``classify_run`` is pure and unit-tested; the build/run +glue shells out and drives hardware, so it is integration-only. +""" + +import os +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Optional, Tuple + +from ntfc.fuzz.mocks import append_mock_defines, missing_board_constants + +if TYPE_CHECKING: + from ntfc.builder import NuttXBuilder + from ntfc.fuzz.profiles import ArchProfile + +PASS = "pass" +PASS_MOCKED = "pass-mocked" +BUILD_FAIL = "build-fail" +TEST_FAIL = "test-fail" +CRASH = "crash" +TIMEOUT = "timeout" +STATUSES = (PASS, PASS_MOCKED, BUILD_FAIL, TEST_FAIL, CRASH, TIMEOUT) + + +@dataclass +class Outcome: + """The result of evaluating one candidate configuration.""" + + configs: FrozenSet[str] + status: str + detail: str = "" + log_path: Optional[str] = None + elf_path: Optional[str] = None + mocks: Optional[Dict[str, str]] = None + + @property + def ok(self) -> bool: + """Whether the candidate passed (natively or after mocking).""" + return self.status in (PASS, PASS_MOCKED) + + +def classify_run( + output: str, rc: int, timed_out: bool, patterns: Dict[str, Any] +) -> Tuple[str, str]: + """Map captured run output + exit code to a ``(status, detail)`` pair.""" + if timed_out: + return TIMEOUT, "no completion within timeout (hang)" + m = re.search(patterns["ostest_exit"], output) + if m: + code = int(m.group(1)) + if code == 0: + return PASS, "" + return TEST_FAIL, f"ostest status {code}" + if re.search(patterns["crash"], output, re.IGNORECASE): + return CRASH, "assertion/panic in output" + if rc == -11: + return CRASH, "segfault (SIGSEGV)" + return CRASH, f"did not finish ostest (exit {rc})" + + +def extract_errors( + log: str, patterns: Dict[str, Any], max_lines: int = 20 +) -> str: + """Pull the most relevant error lines out of a build/run log. + + Lines matching the ``build_error`` patterns are returned; if none match, + the tail of the log is used as a fallback. + """ + rx = re.compile("|".join(patterns["build_error"]), re.IGNORECASE) + hits = [ln.rstrip() for ln in log.splitlines() if rx.search(ln)] + if not hits: + hits = [ln.rstrip() for ln in log.splitlines() if ln.strip()] + hits = hits[-max_lines:] + return "\n".join(hits[:max_lines]) + + +def _config_header(build_dir: str) -> Path: # pragma: no cover + """Path to the build's generated, force-included nuttx/config.h.""" + return Path(build_dir) / "include" / "nuttx" / "config.h" + + +def _save_log( # pragma: no cover + log_dir: str, kind: str, configs: FrozenSet[str], output: str +) -> str: + """Write a build/run log to a file and return its path.""" + os.makedirs(log_dir, exist_ok=True) + tag = "baseline" if not configs else "_".join(sorted(configs)) + path = os.path.join(log_dir, f"{kind}-{tag}.log") + with open(path, "w", encoding="utf-8") as f: + f.write(output) + return path + + +def build_one( # pragma: no cover # noqa: C901 + builder: "NuttXBuilder", + board_config: str, + configs: FrozenSet[str], + build_dir: str, + *, + mock: bool = False, + profile: Optional["ArchProfile"] = None, + jobs: Optional[int] = None, + mock_max_iters: int = 8, + cleanup: bool = True, + log_dir: Optional[str] = None, + patterns: Optional[Dict[str, Any]] = None, +) -> Outcome: + """Build one candidate; classify pass / pass-mocked / build-fail. + + With ``mock`` set, when a build fails the missing board-supplied constants + are synthesised into the build's ``config.h`` and the build retried, up to + ``mock_max_iters`` rounds, so the arch build path can be validated without + a real board wiring. ``cleanup`` removes the build dir when done; the + ostest/mem modes pass ``cleanup=False`` because they use the built ELF. On + a build failure the full compiler log is written to ``log_dir`` and the + first error line becomes the outcome ``detail``. + """ + kv = {c: True for c in configs} + + def fail(res: Any, injected: Optional[Dict[str, str]] = None) -> Outcome: + detail = "build failed" + if patterns is not None: + errs = extract_errors(res.log, patterns) + if errs: + detail = errs.splitlines()[0][:200] + log_path = None + if log_dir is not None: + log_path = _save_log(log_dir, "build", configs, res.log) + return Outcome( + configs, + BUILD_FAIL, + detail, + log_path=log_path, + elf_path=res.elf_path, + mocks=injected or None, + ) + + try: + res = builder.build_candidate(board_config, kv, build_dir, jobs=jobs) + if res.ok: + return Outcome(configs, PASS, elf_path=res.elf_path) + if not mock or profile is None: + return fail(res) + + injected: Dict[str, str] = {} + header = _config_header(build_dir) + for _ in range(mock_max_iters): + new = missing_board_constants(res.log, profile, injected) + if not new: + break + injected.update(new) + append_mock_defines(header, new) + # configure=False: a reconfigure would regenerate config.h and + # discard the mocks appended above + res = builder.build_candidate( + board_config, kv, build_dir, jobs=jobs, configure=False + ) + if res.ok: + return Outcome( + configs, PASS_MOCKED, elf_path=res.elf_path, mocks=injected + ) + return fail(res, injected) + finally: + if cleanup: + shutil.rmtree(build_dir, ignore_errors=True) + + +def run_one( # pragma: no cover + core_conf: Dict[str, Any], + configs: FrozenSet[str], + patterns: Dict[str, Any], + timeout: float, + log_dir: Optional[str] = None, +) -> Outcome: + """Boot a built candidate to NSH, run ``ostest``, and classify it. + + The base config is an NSH target (sim / QEMU / serial). The device is + booted through the device layer, the ``ostest`` builtin is run, and it is + read until the NSH prompt returns (or the timeout). The + ``ostest_main: Exiting with status N`` line is authoritative (0 -> pass, + else test-fail); otherwise a crash signature -> crash, a hang -> timeout, + refined by the device's busy-loop / not-alive state. The console output of + any non-pass run is saved to ``log_dir``. + + ostest under the simulator runs the full suite and is not real-time, so it + can take minutes; give ``timeout`` plenty of headroom. + """ + from ntfc.coreconfig import CoreConfig + from ntfc.device.getdev import get_device + + dev = get_device(CoreConfig(core_conf)) + try: + dev.start() + if not dev._wait_for_boot(int(timeout)): + return Outcome(configs, TIMEOUT, "device did not boot") + res = dev.send_cmd_read_until_pattern( + b"ostest", dev.prompt, int(timeout) + ) + out = res.output + timed_out = res.status.name == "TIMEOUT" + status, detail = classify_run(out, 0, timed_out, patterns) + if status == CRASH: + if dev.busyloop: + status, detail = TIMEOUT, "device busy-loop" + elif dev.notalive: + detail = "device not alive; " + detail + log_path = None + if log_dir is not None and status != PASS: + log_path = _save_log(log_dir, "run", configs, out) + return Outcome(configs, status, detail, log_path=log_path) + finally: + dev.stop() diff --git a/src/ntfc/fuzz/data/arch-profiles.yaml b/src/ntfc/fuzz/data/arch-profiles.yaml new file mode 100644 index 0000000..95b4c8a --- /dev/null +++ b/src/ntfc/fuzz/data/arch-profiles.yaml @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Arch profiles: which Kconfig symbols form each arch's fuzz surface, which are +# never toggled, which build-error identifiers are board-supplied (mockable), +# and which failures are board-config requirements (not code bugs). +# Add a new arch by adding an entry here - no code change required. + +stm32: + # A symbol is in this arch's surface when its defining Kconfig file matches: + symbol_path: 'arch/arm/src/(common/stm32|stm32[^/]*)/' + # Never toggled: capability flags, chip selectors, chip-determined variants. + skip_prefixes: ['STM32_HAVE', 'ARCH_CHIP', 'STM32_FLASH_CONFIG_'] + # Board-supplied constants (mockable in --mock mode): + mockable_prefixes: + ['GPIO_', 'BOARD_', 'DMAMAP_', 'DMACHAN_', 'SDIO_', 'STM32_RCC_'] + mockable_suffixes: ['_CLKIN', '_FREQUENCY'] + # Arch-specific board-config-requirement patterns (merged with generic set): + extra_board_required: + - 'selected HSI48 as USB clock' + - "undefined reference to `(stm32_board_clockconfig|stm32_phy_boardinitialize|mmcsd_slotinitialize|arm_usbinitialize|stm32_spidev_initialize)'" + +sim: + # The POSIX simulator: everything is host-compiled, there are no + # board-provided constants to mock. + symbol_path: 'arch/sim/' + skip_prefixes: ['ARCH_CHIP', 'HOST_'] + mockable_prefixes: [] diff --git a/src/ntfc/fuzz/data/patterns.yaml b/src/ntfc/fuzz/data/patterns.yaml new file mode 100644 index 0000000..0e3692a --- /dev/null +++ b/src/ntfc/fuzz/data/patterns.yaml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Classification regex tables. Editable without touching code. +# - board_required : unmet board/platform prerequisites (not code bugs). +# - config_required: Kconfig dependency problems (fixable with select). +# - build_error : lines pulled as the "error excerpt" from a build log. +# - ostest_exit : capture group 1 is the ostest exit status. +# - crash : any match means an assertion/panic/segfault occurred. + +board_required: + - 'must be defined' + - 'in board\.h' + - 'board\.h needs to define' + - 'Must define .* in' + - 'memory manager regions' + - 'CONFIG_MM_REGIONS' + - "undefined reference to `(?:board_|up_|host_|sim_)\\w+'" + +config_required: + - 'requires CONFIG_\w+' + - 'CONFIG_\w+ (?:is required|must be (?:set|selected|enabled|defined))' + - '#error[^\n]*\brequires\b' + - '#error[^\n]*\bmust be (?:set|selected|enabled)\b' + +build_error: + - 'error:' + - 'fatal error:' + - 'undefined reference' + - 'No such file or directory' + - 'Error \d' + - 'CMake Error' + - 'ninja: error' + - 'recipe for target' + +ostest_exit: 'ostest_main: Exiting with status (-?\d+)' + +crash: 'assert|panic|Assertion failed|up_assert|Segmentation|Backtrace' diff --git a/src/ntfc/fuzz/data/scopes.yaml b/src/ntfc/fuzz/data/scopes.yaml new file mode 100644 index 0000000..a55afa7 --- /dev/null +++ b/src/ntfc/fuzz/data/scopes.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Fuzz scopes: subsystem name -> regex matched against a symbol's defining +# Kconfig file path (top-relative). Orthogonal to the arch profile (which only +# decides how candidates are built/mocked). Add a line to add a subsystem. + +sched: '(?:^|/)sched/' +mm: '(?:^|/)mm/' +fs: '(?:^|/)fs/' +net: '(?:^|/)net/' +binfmt: '(?:^|/)binfmt/' +crypto: '(?:^|/)crypto/' +syscall: '(?:^|/)syscall/' +openamp: '(?:^|/)openamp/' +audio: '(?:^|/)(?:audio|drivers/audio)/' +graphics: '(?:^|/)(?:graphics|drivers/lcd|drivers/video)/' +video: '(?:^|/)(?:video|drivers/video)/' +drivers: '(?:^|/)drivers/' +usb: '(?:^|/)drivers/usb(?:dev|host|misc|monitor)/' +sensors: '(?:^|/)drivers/sensors/' +power: '(?:^|/)drivers/power/' +mtd: '(?:^|/)drivers/mtd/' +input: '(?:^|/)drivers/input/' +serial: '(?:^|/)drivers/serial/' +can: '(?:^|/)drivers/can/' +motor: '(?:^|/)drivers/motor/' +timers: '(?:^|/)drivers/timers/' +wireless: '(?:^|/)(?:wireless|drivers/wireless)/' +kernel: '(?:^|/)(?:sched|mm|fs|net|binfmt|crypto|syscall|openamp)/' diff --git a/src/ntfc/fuzz/dataload.py b/src/ntfc/fuzz/dataload.py new file mode 100644 index 0000000..5af28ec --- /dev/null +++ b/src/ntfc/fuzz/dataload.py @@ -0,0 +1,106 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Load the fuzzer's externalized data tables into runtime objects. + +The fuzzer's slowly-changing knowledge (arch profiles, scope vocabulary, +classification regexes) lives in editable YAML files under ``data/``, shipped +as package data. This module reads them and produces the objects the engine +consumes, so no engine code carries literal tables. Pass ``data_dir`` to load +an alternative set (e.g. a campaign-supplied override directory). +""" + +import re +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml # type: ignore + +from ntfc.fuzz.profiles import ArchProfile + +DATA_DIR = Path(__file__).resolve().parent / "data" + + +class FuzzDataError(Exception): + """Invalid or missing fuzz data file.""" + + +def _read(name: str, data_dir: Optional[Path]) -> Any: + path = (data_dir or DATA_DIR) / name + try: + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + except FileNotFoundError as exc: + raise FuzzDataError(f"fuzz data file not found: {path}") from exc + + +def load_patterns(data_dir: Optional[Path] = None) -> Dict[str, Any]: + """Load the classification regex tables.""" + data: Dict[str, Any] = _read("patterns.yaml", data_dir) + for key in ( + "board_required", + "config_required", + "build_error", + "ostest_exit", + "crash", + ): + if key not in data: + raise FuzzDataError(f"patterns.yaml missing key '{key}'") + return data + + +def load_scopes(data_dir: Optional[Path] = None) -> Dict[str, str]: + """Load the subsystem -> path-regex scope vocabulary.""" + return dict(_read("scopes.yaml", data_dir)) + + +def load_profiles( + data_dir: Optional[Path] = None, +) -> Dict[str, ArchProfile]: + """Load the arch profile registry, merging in the generic patterns.""" + raw = _read("arch-profiles.yaml", data_dir) + pats = load_patterns(data_dir) + gbr = tuple(pats["board_required"]) + gcr = tuple(pats["config_required"]) + out: Dict[str, ArchProfile] = {} + for name, spec in raw.items(): + out[name] = ArchProfile( + name=name, + symbol_path_re=re.compile(spec["symbol_path"]), + skip_prefixes=tuple(spec.get("skip_prefixes", ())), + mockable_prefixes=tuple(spec.get("mockable_prefixes", ())), + mockable_suffixes=tuple(spec.get("mockable_suffixes", ())), + extra_board_required=tuple(spec.get("extra_board_required", ())), + generic_board_required=gbr, + generic_config_required=gcr, + ) + return out + + +def get_profile(name: str, data_dir: Optional[Path] = None) -> ArchProfile: + """Return one arch profile by name, or raise listing the known names.""" + profiles = load_profiles(data_dir) + try: + return profiles[name] + except KeyError as exc: + known = ", ".join(sorted(profiles)) + raise FuzzDataError( + f"unknown arch '{name}'; known profiles: {known}" + ) from exc diff --git a/src/ntfc/fuzz/discover.py b/src/ntfc/fuzz/discover.py new file mode 100644 index 0000000..117af63 --- /dev/null +++ b/src/ntfc/fuzz/discover.py @@ -0,0 +1,178 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Kconfig symbol discovery via kconfiglib. + +Configures a base board with Make so the apps preconfig and arch/board Kconfig +symlinks exist, then parses the tree to enumerate fuzzable symbols. The +environment setup mirrors ``tools/checkkconfig.py`` (the proven recipe). Ported +from the standalone fuzzer; the NuttX checkout is passed as a plain path +instead of the fuzzer's ``Repo`` object. +""" + +import contextlib +import os +import re # noqa: F401 (used in string type annotations) +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING, Iterator, List, Optional + +from ntfc.log.logger import logger + +try: + from kconfiglib import BOOL, Kconfig # type: ignore[import-untyped] +except ImportError: # pragma: no cover + logger.error( + "the fuzz feature depends on kconfiglib. Install it: " + "pip install kconfiglib" + ) + raise + +from ntfc.fuzz.model import SymbolInfo + +if TYPE_CHECKING: + from ntfc.fuzz.profiles import ArchProfile + + +def _run( + cmd: List[str], cwd: Path +) -> "subprocess.CompletedProcess[str]": # pragma: no cover + return subprocess.run(cmd, cwd=cwd, text=True, capture_output=True) + + +@contextlib.contextmanager +def kconfig_env( # pragma: no cover + nuttx_root: Path, board_config: str +) -> Iterator["Kconfig"]: + """Configure the board, yield a loaded Kconfig, then distclean.""" + nuttx_root = Path(nuttx_root).resolve() + r = _run( + [str(nuttx_root / "tools" / "configure.sh"), "-E", board_config], + nuttx_root, + ) + if r.returncode != 0: + if (nuttx_root / ".config").exists(): + _run(["make", "distclean"], nuttx_root) + raise RuntimeError( + f"configure.sh failed for {board_config}\n" f"{r.stdout}{r.stderr}" + ) + + saved = { + k: os.environ.get(k) + for k in ( + "APPSDIR", + "APPSBINDIR", + "EXTERNALDIR", + "BINDIR", + "KCONFIG_CONFIG", + ) + } + os.environ["APPSDIR"] = "../apps" + os.environ["APPSBINDIR"] = "../apps" + os.environ["EXTERNALDIR"] = "dummy" + os.environ["BINDIR"] = str(nuttx_root) + os.environ["KCONFIG_CONFIG"] = str(nuttx_root / ".config") + + cwd = os.getcwd() + try: + os.chdir(nuttx_root) + kconf = Kconfig("Kconfig", warn=True, warn_to_stderr=False) + kconf.load_config() + yield kconf + finally: + os.chdir(cwd) + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + if (nuttx_root / ".config").exists(): + _run(["make", "distclean"], nuttx_root) + + +def _in_scope(sym: object, selector: "re.Pattern[str]") -> bool: + return any( + selector.search(n.filename or "") + for n in sym.nodes # type: ignore[attr-defined] + ) + + +def discover_fuzzable( + kconf: "Kconfig", + profile: "ArchProfile", + selector: "re.Pattern[str]", + include_choices: bool = False, +) -> List[SymbolInfo]: + """Return settable bool symbols in the selector's scope. + + A symbol qualifies when it is a ``bool`` in ``selector``'s scope, is not a + skipped-prefix capability flag, is currently ``n``, and can be set to ``y`` + in the expanded base config. + """ + out: List[SymbolInfo] = [] + for sym in kconf.unique_defined_syms: + if not _in_scope(sym, selector): + continue + if sym.type != BOOL: + continue + if sym.name.startswith(profile.skip_prefixes): + continue + if sym.choice is not None and not include_choices: + continue + if not any(n.prompt for n in sym.nodes): + continue + if sym.str_value != "n": + continue + if 2 not in sym.assignable: # 2 == y + continue + out.append( + SymbolInfo( + name=sym.name, + kind="bool", + cur_value=sym.str_value, + target="y", + files=sorted({n.filename for n in sym.nodes if n.filename}), + in_choice=sym.choice is not None, + ) + ) + out.sort(key=lambda s: s.name) + return out + + +def discover( # pragma: no cover + nuttx_root: Path, + board_config: str, + profile: "ArchProfile", + selector: "re.Pattern[str]", + include_choices: bool = False, + only: Optional[List[str]] = None, +) -> List[SymbolInfo]: + """Configure the board, discover fuzzable symbols, optionally filter.""" + with kconfig_env(nuttx_root, board_config) as kconf: + syms = discover_fuzzable( + kconf, + profile=profile, + selector=selector, + include_choices=include_choices, + ) + if only: + wanted = set(only) + syms = [s for s in syms if s.name in wanted] + return syms diff --git a/src/ntfc/fuzz/engine.py b/src/ntfc/fuzz/engine.py new file mode 100644 index 0000000..3ba71ca --- /dev/null +++ b/src/ntfc/fuzz/engine.py @@ -0,0 +1,511 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Fuzz engine: turn a campaign into build/run/size candidates and a report. + +The three feature modes (``build``, ``ostest``, ``mem``) share the same shape: +resolve the fuzz surface, generate candidate subsets, evaluate each through +NTFC's builder / device layer, and collect a ``FuzzReport``. The build/run/size +steps are injectable so the orchestration is unit-tested with fakes; the +defaults are the real integration functions. +""" + +import itertools +import os +import shutil +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path # noqa: TC003 +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Sequence + +from ntfc.builder import NuttXBuilder +from ntfc.fuzz import candidate, dataload, discover, memsize +from ntfc.fuzz.campaign import ( + CampaignError, + FuzzConfig, + board_config_of, + build_target_conf, +) +from ntfc.fuzz.candidate import ( + BUILD_FAIL, + CRASH, + TEST_FAIL, + TIMEOUT, + Outcome, +) +from ntfc.fuzz.features import Feature, parse_features +from ntfc.fuzz.profiles import build_selector +from ntfc.fuzz.report import FuzzReport +from ntfc.fuzz.strategy import ( + ddmin, + random_subsets, + single_subsets, + systematic_subsets, +) +from ntfc.log.logger import logger + +_RUNTIME_FAILS = (CRASH, TEST_FAIL, TIMEOUT) + +# Progress callback: called with a human-readable step message. Defaults to a +# no-op; the CLI passes a printer so users see live progress during long runs. +Progress = Callable[[str], None] + + +def _noop(_msg: str) -> None: + pass + + +def _label(configs: FrozenSet[str]) -> str: + return "baseline" if not configs else "{" + ",".join(sorted(configs)) + "}" + + +def _emit_outcome(emit: Progress, prefix: str, o: Outcome) -> None: + """Emit a candidate's status with its detail and saved log path.""" + detail = f" -- {o.detail}" if o.detail else "" + emit(f"{prefix}{o.status}{detail}") + if o.log_path: + emit(f"[fuzz] log saved: {o.log_path}") + + +def _cfg(name: str) -> str: + """Ensure a symbol name carries the ``CONFIG_`` prefix.""" + return name if name.startswith("CONFIG_") else f"CONFIG_{name}" + + +def _map( + items: Sequence[Any], fn: Callable[[Any], Any], run: Dict[str, Any] +) -> List[Any]: + """Evaluate ``fn`` over items, in parallel or sequentially per ``run``.""" + if run.get("parallel"): + workers = max(1, int(run.get("workers", 1))) + with ThreadPoolExecutor(max_workers=workers) as pool: + return list(pool.map(fn, items)) + return [fn(i) for i in items] + + +def _subsets( + strategy: Dict[str, Any], names: Sequence[str] +) -> List[FrozenSet[str]]: + """Generate candidate subsets for the configured strategy mode. + + ``strategy.limit`` caps the candidate count (0 / absent = no cap). + """ + mode = strategy.get("mode", "single") + if mode == "single": + subs = single_subsets(names) + elif mode == "random": + subs = random_subsets( + list(names), + int(strategy.get("rounds", 20)), + int(strategy.get("size", 4)), + int(strategy.get("seed", 0)), + ) + else: + subs = systematic_subsets(names, mode) + limit = int(strategy.get("limit", 0)) + if limit and len(subs) > limit: + logger.info( + "[fuzz] limiting %d candidates to limit=%d", len(subs), limit + ) + subs = subs[:limit] + return subs + + +def _run_opts(fuzz: FuzzConfig) -> Dict[str, Any]: + """Parallelism options for ``_map`` from the fuzz config.""" + return {"parallel": fuzz.parallel, "workers": fuzz.workers} + + +def _surface_names( + fuzz: FuzzConfig, + nuttx_root: Path, + profile: Any, + board_config: str, + discover_fn: Callable[..., Any], +) -> List[str]: + """Flat list of ``CONFIG_`` option names for the build sweep surface.""" + s = fuzz.surface + if s.get("symbols"): + return [_cfg(n) for n in s["symbols"]] + if s.get("features"): + names: List[str] = [] + for configs in s["features"].values(): + names.extend(_cfg(c) for c in configs) + return names + scope = s["scope"] + scope_arg = ",".join(scope) if isinstance(scope, list) else scope + selector = build_selector(profile, scope_arg, dataload.load_scopes()) + syms = discover_fn( + nuttx_root, + board_config, + profile, + selector, + include_choices=s.get("include_choices", False), + ) + return [_cfg(sym.name) for sym in syms] + + +def _features_from_surface(surface: Dict[str, Any], require: List[str]) -> Any: + """Build the ostest/mem feature model from a symbols/features surface.""" + req = ",".join(require) if require else None + if surface.get("features"): + specs = [ + f"{name}=" + ",".join(configs) + for name, configs in surface["features"].items() + ] + return parse_features(None, specs, req) + if surface.get("symbols"): + return parse_features(",".join(surface["symbols"]), (), req) + raise CampaignError( + "ostest/mem surface must set 'symbols' or 'features' (not 'scope')" + ) + + +def _mem_variants( + strategy: Dict[str, Any], names: Sequence[str] +) -> List[FrozenSet[str]]: + """Baseline + systematic variants for a mem sweep, capped at max_builds.""" + mode = strategy.get("mode", "marginal") + variants: List[FrozenSet[str]] = [frozenset()] + systematic_subsets( + names, mode if mode in ("marginal", "pairs", "full") else "marginal" + ) + max_builds = int(strategy.get("max_builds", 64)) + if len(variants) > max_builds: + logger.info( + "[fuzz] mem: capping %d variants at max_builds=%d", + len(variants), + max_builds, + ) + variants = variants[:max_builds] + return variants + + +def _build_root(build_dir: str) -> str: + return os.path.join(build_dir, "fuzz") + + +def plan_candidates( + fuzz: FuzzConfig, + target: Optional[Dict[str, Any]] = None, + *, + discover_fn: Callable[..., Any] = discover.discover, +) -> List[FrozenSet[str]]: + """Return the candidate subsets a campaign would evaluate (no builds). + + This is the ``--dry-run`` matrix. ``build`` sweeps candidate config + subsets directly; ``ostest`` and ``mem`` sweep feature-name subsets and + include the baseline (the empty set) the engine evaluates first. + """ + if fuzz.feature == "build": + profile = dataload.get_profile(fuzz.arch) + nuttx_root = Path(str(fuzz.tree)) / "nuttx" + names = _surface_names( + fuzz, nuttx_root, profile, str(fuzz.board), discover_fn + ) + return _subsets(fuzz.strategy, names) + feats, _req = _features_from_surface(fuzz.surface, fuzz.require) + names = [f.name for f in feats] + if fuzz.feature == "mem": + return _mem_variants(fuzz.strategy, names) + subsets = _subsets(fuzz.strategy, names) + if not fuzz.strategy.get("skip_baseline"): + subsets.insert(0, frozenset()) + return subsets + + +def run_build( + fuzz: FuzzConfig, + *, + discover_fn: Callable[..., Any] = discover.discover, + build_fn: Callable[..., Outcome] = candidate.build_one, + progress: Optional[Progress] = None, +) -> FuzzReport: + """Build-break sweep: build each candidate, classify pass / build-fail. + + ``build`` needs no external target: the fuzz config names the ``board`` and + the ``tree`` (dir with ``nuttx/`` and ``apps/``) to build. + """ + emit = progress or _noop + profile = dataload.get_profile(fuzz.arch) + patterns = dataload.load_patterns() + board_config = str(fuzz.board) + base_conf = build_target_conf(str(fuzz.tree), fuzz.build_dir) + nuttx_root = Path(str(fuzz.tree)) / "nuttx" + emit(f"[fuzz] discovering fuzz surface on {board_config} ...") + names = _surface_names( + fuzz, nuttx_root, profile, board_config, discover_fn + ) + subsets = _subsets(fuzz.strategy, names) + builder = NuttXBuilder(base_conf, rebuild=True) + build_root = _build_root(fuzz.build_dir) + logs_dir = os.path.join(build_root, "logs") + report = FuzzReport(base=board_config, feature="build") + total = len(subsets) + emit(f"[fuzz] build sweep: {total} candidate(s) on {board_config}") + done = itertools.count(1) + + def evaluate(item: Any) -> Outcome: + idx, cfgs = item + bd = os.path.join(build_root, f"fuzz-{idx:04d}") + emit(f"[fuzz] building {_label(cfgs)} ...") + o = build_fn( + builder, + board_config, + cfgs, + bd, + mock=fuzz.mock, + profile=profile, + jobs=fuzz.jobs, + log_dir=logs_dir, + patterns=patterns, + ) + _emit_outcome( + emit, f"[fuzz] [{next(done)}/{total}] {_label(cfgs)}: ", o + ) + return o + + for o in _map(list(enumerate(subsets)), evaluate, _run_opts(fuzz)): + report.add(o) + emit("[fuzz] build sweep complete") + return report + + +def run_ostest( # noqa: C901 + fuzz: FuzzConfig, + target: Dict[str, Any], + *, + build_fn: Callable[..., Outcome] = candidate.build_one, + run_fn: Callable[..., Outcome] = candidate.run_one, + progress: Optional[Progress] = None, +) -> FuzzReport: + """Build+run each candidate under ostest, then ddmin the failures. + + ``target`` is a normal NTFC config (from ``--confpath``): it provides the + board defconfig, the device (sim / QEMU / serial) and its run parameters. + """ + emit = progress or _noop + profile = dataload.get_profile(fuzz.arch) + patterns = dataload.load_patterns() + board_config = board_config_of(target) + feats, req = _features_from_surface(fuzz.surface, fuzz.require) + fmap: Dict[str, Feature] = {f.name: f for f in feats} + names = [f.name for f in feats] + builder = NuttXBuilder(target, rebuild=True) + build_root = _build_root(target["config"]["build_dir"]) + counter = itertools.count() + report = FuzzReport(base=board_config, feature="ostest") + + def configs_for(feat_names: FrozenSet[str]) -> FrozenSet[str]: + cfgs = set(req) + for n in feat_names: + cfgs |= fmap[n].configs + return frozenset(cfgs) + + logs_dir = os.path.join(build_root, "logs") + + def evaluate(feat_names: FrozenSet[str]) -> Outcome: + bd = os.path.join(build_root, f"fuzz-{next(counter):04d}") + emit(f"[fuzz] building {_label(feat_names)} ...") + bo = build_fn( + builder, + board_config, + configs_for(feat_names), + bd, + mock=fuzz.mock, + profile=profile, + jobs=fuzz.jobs, + cleanup=False, + log_dir=logs_dir, + patterns=patterns, + ) + try: + if not bo.ok: + out = Outcome( + feat_names, bo.status, bo.detail, log_path=bo.log_path + ) + _emit_outcome(emit, f"[fuzz] {_label(feat_names)}: ", out) + return out + emit( + f"[fuzz] running ostest for {_label(feat_names)} " + f"(up to {fuzz.timeout:g}s) ..." + ) + core_conf = _core_conf(target, bo.elf_path or "") + o = run_fn( + core_conf, + feat_names, + patterns, + fuzz.timeout, + log_dir=logs_dir, + ) + _emit_outcome(emit, f"[fuzz] {_label(feat_names)}: ", o) + return o + finally: + shutil.rmtree(bd, ignore_errors=True) + + if not fuzz.strategy.get("skip_baseline"): + emit("[fuzz] evaluating baseline ...") + report.baseline = evaluate(frozenset()) + if report.baseline.status == BUILD_FAIL: + emit("[fuzz] baseline failed to build; aborting sweep.") + logger.error( + "[fuzz] baseline failed to build; aborting sweep. Check the " + "target config and that /nuttx and /apps exist." + ) + return report + + subsets = _subsets(fuzz.strategy, names) + emit(f"[fuzz] ostest sweep: {len(subsets)} candidate(s) on {board_config}") + for o in _map(subsets, lambda s: evaluate(frozenset(s)), _run_opts(fuzz)): + report.add(o) + + # Only delta-debug genuine runtime failures; a build-fail does not + # compile regardless of the subset, so minimising it is meaningless. + if fuzz.strategy.get("minimize", True): + for o in report.outcomes: + if o.status in _RUNTIME_FAILS and len(o.configs) > 1: + emit(f"[fuzz] minimising {_label(o.configs)} ...") + minimal = ddmin( + sorted(o.configs), + lambda s: evaluate(frozenset(s)).status in _RUNTIME_FAILS, + ) + report.minimized.append((o, frozenset(minimal))) + return report + + +def run_mem( # noqa: C901 + fuzz: FuzzConfig, + *, + build_fn: Callable[..., Outcome] = candidate.build_one, + size_fn: Callable[..., Any] = memsize.elf_size, + progress: Optional[Progress] = None, +) -> FuzzReport: + """Memory sweep: build base + variants, diff flash/RAM footprints. + + Like ``build``, ``mem`` only compiles, so the fuzz config names the + ``board`` and ``tree`` -- no external target config is needed. + """ + emit = progress or _noop + profile = dataload.get_profile(fuzz.arch) + patterns = dataload.load_patterns() + board_config = str(fuzz.board) + base_conf = build_target_conf(str(fuzz.tree), fuzz.build_dir) + feats, req = _features_from_surface(fuzz.surface, fuzz.require) + fmap: Dict[str, Feature] = {f.name: f for f in feats} + names = [f.name for f in feats] + builder = NuttXBuilder(base_conf, rebuild=True) + build_root = _build_root(fuzz.build_dir) + logs_dir = os.path.join(build_root, "logs") + counter = itertools.count() + report = FuzzReport(base=board_config, feature="mem") + + variants = _mem_variants(fuzz.strategy, names) + + def configs_for(feat_names: FrozenSet[str]) -> FrozenSet[str]: + cfgs = set(req) + for n in feat_names: + cfgs |= fmap[n].configs + return frozenset(cfgs) + + def evaluate(feat_names: FrozenSet[str]) -> Any: + bd = os.path.join(build_root, f"mem-{next(counter):04d}") + emit(f"[fuzz] building + sizing {_label(feat_names)} ...") + bo = build_fn( + builder, + board_config, + configs_for(feat_names), + bd, + mock=fuzz.mock, + profile=profile, + jobs=fuzz.jobs, + cleanup=False, + log_dir=logs_dir, + patterns=patterns, + ) + try: + if not bo.ok: + _emit_outcome(emit, f"[fuzz] {_label(feat_names)}: ", bo) + return feat_names, None, bo + text, data, bss = size_fn(bo.elf_path) + return feat_names, memsize.footprint(text, data, bss), bo + finally: + shutil.rmtree(bd, ignore_errors=True) + + emit(f"[fuzz] mem sweep: {len(variants)} variant(s) on {board_config}") + results = _map(variants, lambda s: evaluate(frozenset(s)), _run_opts(fuzz)) + sizes: Dict[FrozenSet[str], Any] = {fn: fp for fn, fp, _ in results} + for fn, _fp, bo in results: + if not fn: + report.baseline = bo + else: + report.add(bo) + base_fp = sizes.get(frozenset()) + if base_fp is None: + logger.error("[fuzz] mem: baseline did not build; no deltas") + return report + + for feat_names in variants: + if not feat_names: + continue + fp = sizes.get(feat_names) + if fp is None: + continue + report.memory.append( + { + "feature": ",".join(sorted(feat_names)), + "flash_delta": fp[0] - base_fp[0], + "ram_delta": fp[1] - base_fp[1], + } + ) + return report + + +def run_campaign( + fuzz: FuzzConfig, + target: Optional[Dict[str, Any]] = None, + **kwargs: Any, +) -> FuzzReport: + """Dispatch a fuzz config to its engine mode. + + ``target`` is the NTFC target config and is required only for ``ostest`` + (build / mem carry their own build target in the fuzz config). + """ + if fuzz.feature == "build": + return run_build(fuzz, **kwargs) + if fuzz.feature == "mem": + return run_mem(fuzz, **kwargs) + if target is None: + raise CampaignError("ostest needs a target config (--confpath)") + return run_ostest(fuzz, target, **kwargs) + + +def _core_conf( # pragma: no cover + base_conf: Dict[str, Any], elf_path: str +) -> Dict[str, Any]: + """Build a single-core NTFC config dict pointing at a built candidate.""" + core: Dict[str, Any] = {} + for key, val in base_conf.items(): + if "product" in key and isinstance(val, dict): + cores = val.get("cores", {}) + if cores: + core = dict(next(iter(cores.values()))) + break + core["elf_path"] = elf_path + core["conf_path"] = os.path.join(os.path.dirname(elf_path), ".config") + core.setdefault("exec_path", elf_path) + return core diff --git a/src/ntfc/fuzz/features.py b/src/ntfc/fuzz/features.py new file mode 100644 index 0000000..6b358e7 --- /dev/null +++ b/src/ntfc/fuzz/features.py @@ -0,0 +1,103 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""The fuzzer feature model. + +A *feature* is a named set of ``CONFIG`` options enabled together and treated +as one unit. It may be a single option (the common case) or a group that only +builds when several options are set together (e.g. NET needs a device). A +shared prerequisite context (``require``) can be enabled in every build. Ported +from the standalone ``kconfmem`` module; ``SystemExit`` is replaced with +``ValueError`` so callers can handle bad input. +""" + +from dataclasses import dataclass +from typing import FrozenSet, List, Optional, Sequence, Tuple + + +@dataclass(frozen=True) +class Feature: + """A named set of CONFIG options treated as one unit.""" + + name: str + configs: FrozenSet[str] + + +def normalize(options: Sequence[str]) -> List[str]: + """Bare option names (strip an optional CONFIG_ prefix), de-duplicated.""" + out: List[str] = [] + for o in options: + o = o.strip() + if not o: + continue + if o.startswith("CONFIG_"): + o = o[len("CONFIG_") :] + if o not in out: + out.append(o) + return out + + +def parse_features( + options: Optional[str], + feature_specs: Sequence[str], + require: Optional[str], +) -> Tuple[List[Feature], FrozenSet[str]]: + """Build the feature list from single-config options and grouped features. + + :param options: comma-separated ``CONFIG_*`` options, each its own feature. + :param feature_specs: ``NAME=CONFIG_A,CONFIG_B`` group strings. + :param require: comma-separated always-on prerequisite options. + :return: ``(features, require_set)``. Configs keep the ``CONFIG_`` prefix. + :raises ValueError: on empty or malformed input. + """ + features: List[Feature] = [] + seen = set() + + def add(name: str, configs: Sequence[str]) -> None: + if name in seen: + raise ValueError(f"duplicate feature name '{name}'") + seen.add(name) + features.append( + Feature(name, frozenset(f"CONFIG_{c}" for c in configs)) + ) + + if options: + for c in normalize(options.split(",")): + add(c if c.startswith("CONFIG_") else f"CONFIG_{c}", [c]) + + for spec in feature_specs: + if "=" in spec: + name, rest = spec.split("=", 1) + name = name.strip() + else: + name, rest = "", spec + configs = normalize(rest.split(",")) + if not configs: + raise ValueError(f"empty feature: '{spec}'") + add(name or f"CONFIG_{configs[0]}", configs) + + req = ( + frozenset(f"CONFIG_{c}" for c in normalize(require.split(","))) + if require + else frozenset() + ) + if not features: + raise ValueError("no options/features given") + return features, req diff --git a/src/ntfc/fuzz/memsize.py b/src/ntfc/fuzz/memsize.py new file mode 100644 index 0000000..3bb7bf4 --- /dev/null +++ b/src/ntfc/fuzz/memsize.py @@ -0,0 +1,54 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Linked-image size measurement for the memory-footprint fuzz mode. + +Parses ``size `` (Berkeley format) into ``(text, data, bss)`` and derives +flash/RAM: ``flash = text + data``, ``ram = data + bss``. This is the static +footprint of the linked image; it does not model runtime heap or stack. +""" + +import subprocess +from typing import Tuple + + +def elf_size(elf_path: str, size_tool: str = "size") -> Tuple[int, int, int]: + """Return ``(text, data, bss)`` for an ELF, or zeros if size fails.""" + try: + r = subprocess.run( + [size_tool, elf_path], text=True, capture_output=True + ) + except OSError: + return (0, 0, 0) + if r.returncode != 0: + return (0, 0, 0) + rows = [ln for ln in r.stdout.splitlines() if ln.strip()] + if len(rows) < 2: + return (0, 0, 0) + parts = rows[1].split() + try: + return (int(parts[0]), int(parts[1]), int(parts[2])) + except (ValueError, IndexError): + return (0, 0, 0) + + +def footprint(text: int, data: int, bss: int) -> Tuple[int, int]: + """Return ``(flash, ram)`` from section sizes.""" + return (text + data, data + bss) diff --git a/src/ntfc/fuzz/mocks.py b/src/ntfc/fuzz/mocks.py new file mode 100644 index 0000000..f69bca0 --- /dev/null +++ b/src/ntfc/fuzz/mocks.py @@ -0,0 +1,112 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Board-constant mocking for --mock mode. + +A board's ``include/board.h`` normally defines the concrete pin mappings and +geometry constants the arch drivers consume. When a peripheral is enabled on a +board that does not wire it, those constants are missing and the arch path +never compiles. This module reads the build errors and synthesises just those +*board-supplied* constants (identified by the ArchProfile's mockable +prefixes/suffixes), so the arch build path can be validated. CONFIG_*, type +names, and local variables are never mocked, so real code bugs still fail. +""" + +import re +from typing import TYPE_CHECKING, Dict, Optional, Tuple + +if TYPE_CHECKING: + from pathlib import Path + + from ntfc.fuzz.profiles import ArchProfile + +# error: 'GPIO_CAN2_RX' undeclared (...); did you mean 'GPIO_CAN2_RX_1'? +_UNDECLARED_RE = re.compile( + r"error: '([A-Za-z_]\w*)' undeclared" + r"(?:[^\n;]*; did you mean '([A-Za-z_]\w*)'\?)?" +) + +# #error BOARD_LTDC_WIDTH must be defined in the board.h header file +_MUSTDEF_RE = re.compile(r"#error\s+\"?([A-Z_][A-Z0-9_]*) must be defined") + + +def parse_mock_defines( + log: str, + existing: Dict[str, str], + prefixes: Tuple[str, ...], + suffixes: Tuple[str, ...] = (), +) -> Dict[str, str]: + """Return the new board constants to define, given a build log. + + Only identifiers starting with one of ``prefixes`` or ending with one of + ``suffixes`` are considered board-supplied and mockable. For an undeclared + constant, use gcc's "did you mean" suggestion when it is an alternative of + the same symbol (e.g. ``GPIO_CAN2_RX`` -> ``GPIO_CAN2_RX_1``); otherwise + define it to 0. "must be defined" geometry constants get 1. + """ + + def mockable(name: str) -> bool: + return name.startswith(prefixes) or name.endswith(suffixes) + + new: Dict[str, str] = {} + for m in _UNDECLARED_RE.finditer(log): + name, suggest = m.group(1), m.group(2) + if not mockable(name) or name in existing or name in new: + continue + if suggest and suggest.startswith(name): + new[name] = suggest + else: + new[name] = "0" + for m in _MUSTDEF_RE.finditer(log): + name = m.group(1) + if not mockable(name) or name in existing or name in new: + continue + new[name] = "1" + return new + + +def missing_board_constants( + log: str, + profile: "ArchProfile", + existing: Optional[Dict[str, str]] = None, +) -> Dict[str, str]: + """Mock the board constants a build log needs, per the profile.""" + return parse_mock_defines( + log, + existing or {}, + profile.mockable_prefixes, + profile.mockable_suffixes, + ) + + +def append_mock_defines(config_h: "Path", defines: Dict[str, str]) -> None: + """Write a block of ``#define``s to the build's generated nuttx/config.h. + + config.h is force-included by every translation unit and lives in the build + directory, so this injects the mocks without touching any source file. + """ + if not defines: + return + lines = ["", "/* ntfc fuzz: mocked board constants */"] + for name in sorted(defines): + lines.append(f"#define {name} {defines[name]}") + lines.append("") + with config_h.open("a") as f: + f.write("\n".join(lines)) diff --git a/src/ntfc/fuzz/model.py b/src/ntfc/fuzz/model.py new file mode 100644 index 0000000..2ac0763 --- /dev/null +++ b/src/ntfc/fuzz/model.py @@ -0,0 +1,40 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Data model shared across the fuzz modules.""" + +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List + + +@dataclass +class SymbolInfo: + """A fuzzable Kconfig symbol discovered for a base configuration.""" + + name: str # symbol name without the CONFIG_ prefix + kind: str # "bool" (only bool is fuzzed in this version) + cur_value: str # value in the expanded base config ("n", "y", ...) + target: str # value the candidate sets it to ("y") + files: List[str] = field(default_factory=list) # defining Kconfig files + in_choice: bool = False # member of a choice block + + def to_dict(self) -> Dict[str, Any]: + """Return the symbol as a plain dictionary.""" + return asdict(self) diff --git a/src/ntfc/fuzz/profiles.py b/src/ntfc/fuzz/profiles.py new file mode 100644 index 0000000..b7bcc62 --- /dev/null +++ b/src/ntfc/fuzz/profiles.py @@ -0,0 +1,96 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Architecture profiles for the Kconfig fuzzer. + +All architecture-specific knowledge lives in an ``ArchProfile``: which Kconfig +symbols form the fuzz surface, which are never toggled, which build-error +identifiers are board-supplied (mockable), and which failures are +board-configuration requirements rather than code bugs. The profile data comes +from ``data/arch-profiles.yaml`` via :mod:`ntfc.fuzz.dataload`; this module +only defines the runtime object and the scope-selector compiler. +""" + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + + +@dataclass(frozen=True) +class ArchProfile: + """Rules describing one architecture's fuzz surface.""" + + name: str + symbol_path_re: "re.Pattern[str]" + skip_prefixes: Tuple[str, ...] + mockable_prefixes: Tuple[str, ...] + mockable_suffixes: Tuple[str, ...] = field(default_factory=tuple) + extra_board_required: Tuple[str, ...] = field(default_factory=tuple) + generic_board_required: Tuple[str, ...] = field(default_factory=tuple) + generic_config_required: Tuple[str, ...] = field(default_factory=tuple) + + def board_required_re(self) -> "re.Pattern[str]": + """Patterns that mean a board-config prerequisite is unmet.""" + pats = list(self.generic_board_required) + list( + self.extra_board_required + ) + return re.compile("|".join(pats), re.IGNORECASE) + + def config_required_re(self) -> "re.Pattern[str]": + """Patterns that mean a Kconfig dependency is unmet.""" + return re.compile( + "|".join(self.generic_config_required), re.IGNORECASE + ) + + def is_arch_symbol(self, sym: object) -> bool: + """Whether a kconfiglib symbol belongs to this arch's surface.""" + return any( + self.symbol_path_re.search(n.filename or "") + for n in sym.nodes # type: ignore[attr-defined] + ) + + +def build_selector( + profile: ArchProfile, + scope_arg: Optional[str], + scopes: Dict[str, str], +) -> "re.Pattern[str]": + """Compile the symbol selector for a scope argument. + + ``None`` -> the arch profile's own surface. ``'arch'`` -> the arch surface; + ``'all'`` -> everything; otherwise a comma-separated list of named scopes + (e.g. ``'net,fs,audio'``) matched against a symbol's defining Kconfig file. + """ + from ntfc.fuzz.dataload import FuzzDataError + + if not scope_arg: + return profile.symbol_path_re + pats: List[str] = [] + for name in (s.strip() for s in scope_arg.split(",")): + if name == "all": + return re.compile(".") + if name == "arch": + pats.append(profile.symbol_path_re.pattern) + elif name in scopes: + pats.append(scopes[name]) + else: + known = ", ".join(["arch", "all"] + sorted(scopes)) + raise FuzzDataError(f"unknown scope '{name}'; known: {known}") + return re.compile("|".join(pats)) diff --git a/src/ntfc/fuzz/report.py b/src/ntfc/fuzz/report.py new file mode 100644 index 0000000..2531103 --- /dev/null +++ b/src/ntfc/fuzz/report.py @@ -0,0 +1,145 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Fuzz run reporting: categorized text + JSON, written to the session dir.""" + +import json +import os +from dataclasses import dataclass, field +from typing import Any, Dict, FrozenSet, List, Optional, Tuple + +from ntfc.fuzz.candidate import STATUSES, Outcome + + +def _label(configs: FrozenSet[str]) -> str: + return "baseline" if not configs else "{" + ",".join(sorted(configs)) + "}" + + +@dataclass +class FuzzReport: + """Collects candidate outcomes and renders text / JSON reports.""" + + base: str + feature: str + outcomes: List[Outcome] = field(default_factory=list) + baseline: Optional[Outcome] = None + minimized: List[Tuple[Outcome, FrozenSet[str]]] = field( + default_factory=list + ) + memory: List[Dict[str, Any]] = field(default_factory=list) + + def add(self, outcome: Outcome) -> None: + """Record one candidate outcome.""" + self.outcomes.append(outcome) + + def counts(self) -> Dict[str, int]: + """Return per-status counts across all outcomes.""" + c = {s: 0 for s in STATUSES} + for o in self.outcomes: + c[o.status] = c.get(o.status, 0) + 1 + return c + + def failed(self) -> bool: + """Whether any candidate did not pass (natively or mocked).""" + return any(not o.ok for o in self.outcomes) + + def _outcome_dict(self, o: Outcome) -> Dict[str, Any]: + return { + "configs": sorted(o.configs), + "status": o.status, + "detail": o.detail, + "log_path": o.log_path, + "mocks": o.mocks, + } + + def json(self) -> Dict[str, Any]: + """Return the report as a JSON-serialisable dictionary.""" + return { + "base": self.base, + "feature": self.feature, + "counts": self.counts(), + "baseline": ( + self._outcome_dict(self.baseline) + if self.baseline is not None + else None + ), + "outcomes": [self._outcome_dict(o) for o in self.outcomes], + "minimized": [ + {"original": sorted(o.configs), "minimal": sorted(m)} + for o, m in self.minimized + ], + "memory": self.memory, + } + + def text(self) -> str: # noqa: C901 + """Return a human-readable text report.""" + out: List[str] = [] + out.append(f"[fuzz:{self.feature}] base {self.base}") + if self.baseline is not None: + det = f" ({self.baseline.detail})" if self.baseline.detail else "" + out.append(f"[fuzz] baseline: {self.baseline.status}{det}") + counts = self.counts() + out.append( + "[fuzz] outcomes: " + + ", ".join(f"{s}={counts[s]}" for s in STATUSES) + ) + out.append("") + + fails = [o for o in self.outcomes if not o.ok] + if fails: + out.append("failing candidates:") + for o in fails: + line = f" {_label(o.configs):<44} {o.status}" + if o.detail: + line += f" ({o.detail})" + if o.log_path: + line += f" {o.log_path}" + out.append(line) + out.append("") + if self.minimized: + out.append("minimised failures (delta-debug):") + for o, m in self.minimized: + out.append(f" {_label(o.configs)} -> {_label(m)}") + out.append("") + if self.memory: + out.append("memory (flash/ram delta vs baseline):") + for row in self.memory: + out.append( + f" {row['feature']:<32} " + f"flash {row['flash_delta']:+d} ram {row['ram_delta']:+d}" + ) + out.append("") + if not fails and not self.memory: + out.append("all candidates passed.") + return "\n".join(out) + + def write(self, session_dir: str) -> None: + """Write ``fuzz-report.txt`` and ``fuzz-report.json`` to the dir.""" + os.makedirs(session_dir, exist_ok=True) + with open( + os.path.join(session_dir, "fuzz-report.txt"), "w", encoding="utf-8" + ) as f: + f.write(self.text() + "\n") + with open( + os.path.join(session_dir, "fuzz-report.json"), + "w", + encoding="utf-8", + ) as f: + json.dump(self.json(), f, indent=2) diff --git a/src/ntfc/fuzz/strategy.py b/src/ntfc/fuzz/strategy.py new file mode 100644 index 0000000..8edc8e9 --- /dev/null +++ b/src/ntfc/fuzz/strategy.py @@ -0,0 +1,110 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +"""Candidate subset generation and delta-debug minimization. + +Pure functions over lists of names (symbols or feature names): the +single-option sweep, random subsets, systematic subsets (marginal / pairs / +full), and ``ddmin`` to reduce a failing subset to a minimal one. Ported from +the standalone fuzzer; baseline handling is left to the caller, and +``SystemExit`` is replaced with ``ValueError``. +""" + +import itertools +import random +from typing import Callable, FrozenSet, List, Sequence + + +def single_subsets(names: Sequence[str]) -> List[FrozenSet[str]]: + """Return one singleton subset per name (the build-break sweep).""" + return [frozenset([n]) for n in names] + + +def random_subsets( + names: Sequence[str], rounds: int, size: int, seed: int +) -> List[FrozenSet[str]]: + """Return ``rounds`` random subsets of 1..``size`` names, seeded.""" + rng = random.Random(seed) + subs: List[FrozenSet[str]] = [] + names = list(names) + hi = max(1, min(size, len(names))) + for _ in range(rounds): + k = rng.randint(1, hi) + subs.append(frozenset(rng.sample(names, k))) + return subs + + +def systematic_subsets( + names: Sequence[str], mode: str +) -> List[FrozenSet[str]]: + """Deterministic subsets for a systematic mode (no baseline included). + + marginal: each name alone, plus the all-on set. + pairs: each name alone, plus every unordered pair. + full: every non-empty subset. + """ + names = list(names) + subs: List[FrozenSet[str]] = [] + if mode == "marginal": + subs = [frozenset([n]) for n in names] + if len(names) > 1: + subs.append(frozenset(names)) + elif mode == "pairs": + subs = [frozenset([n]) for n in names] + subs += [frozenset(c) for c in itertools.combinations(names, 2)] + elif mode == "full": + for r in range(1, len(names) + 1): + subs += [frozenset(c) for c in itertools.combinations(names, r)] + else: + raise ValueError(f"unknown mode '{mode}'") + + seen = set() + uniq: List[FrozenSet[str]] = [] + for s in subs: + if s not in seen: + seen.add(s) + uniq.append(s) + return uniq + + +def ddmin( + items: Sequence[str], fails: Callable[[List[str]], bool] +) -> List[str]: + """Delta-debug to a minimal failing subset. + + ``fails(subset)`` returns True when the subset still reproduces the + failure. Returns the smallest sublist for which ``fails`` stays True. + """ + items = list(items) + n = 2 + while len(items) > 1: + size = max(1, len(items) // n) + chunks = [items[i : i + size] for i in range(0, len(items), size)] + for ch in chunks: + complement = [x for x in items if x not in ch] + if complement and fails(complement): + items = complement + n = max(n - 1, 2) + break + else: + if n >= len(items): + break + n = min(len(items), n * 2) + return items diff --git a/tests/fuzz/__init__.py b/tests/fuzz/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fuzz/conftest.py b/tests/fuzz/conftest.py new file mode 100644 index 0000000..c937ce3 --- /dev/null +++ b/tests/fuzz/conftest.py @@ -0,0 +1,35 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import pytest + + +@pytest.fixture(autouse=True) +def _isolated_build_root(tmp_path, monkeypatch): + """Keep engine tests away from the repo's real ./build directory. + + The engines rmtree their per-candidate build dirs; without this, a test + run would delete directories of a real fuzz sweep happening in the same + checkout (the campaign resources use the default ``./build``). + """ + monkeypatch.setattr( + "ntfc.fuzz.engine._build_root", + lambda build_dir: str(tmp_path / "fuzz"), + ) diff --git a/tests/fuzz/resources/campaign-build.yaml b/tests/fuzz/resources/campaign-build.yaml new file mode 100644 index 0000000..905cb86 --- /dev/null +++ b/tests/fuzz/resources/campaign-build.yaml @@ -0,0 +1,13 @@ +feature: build +arch: stm32 +board: sim:ntfc +tree: ./external +surface: + scope: [can] + include_choices: false +strategy: + mode: single +mock: true +parallel: true +workers: 2 +jobs: 4 diff --git a/tests/fuzz/resources/campaign-mem.yaml b/tests/fuzz/resources/campaign-mem.yaml new file mode 100644 index 0000000..d5c1d13 --- /dev/null +++ b/tests/fuzz/resources/campaign-mem.yaml @@ -0,0 +1,10 @@ +feature: mem +arch: stm32 +board: sim:ntfc +tree: ./external +surface: + symbols: [FS_FAT, CRYPTO] +strategy: + mode: marginal +workers: 2 +jobs: 4 diff --git a/tests/fuzz/resources/campaign-ostest.yaml b/tests/fuzz/resources/campaign-ostest.yaml new file mode 100644 index 0000000..a0837b8 --- /dev/null +++ b/tests/fuzz/resources/campaign-ostest.yaml @@ -0,0 +1,13 @@ +feature: ostest +arch: stm32 +surface: + symbols: [BAD, HARMLESS_A, HARMLESS_B] +strategy: + mode: random + rounds: 6 + size: 3 + seed: 1 + minimize: true +timeout: 60 +workers: 2 +jobs: 4 diff --git a/tests/fuzz/resources/target-sim.yaml b/tests/fuzz/resources/target-sim.yaml new file mode 100644 index 0000000..f8cb2c3 --- /dev/null +++ b/tests/fuzz/resources/target-sim.yaml @@ -0,0 +1,10 @@ +config: + cwd: './external' + build_dir: './build' +product: + name: "fuzz-sim" + cores: + core0: + name: 'main' + device: 'sim' + defconfig: 'boards/sim/sim/sim/configs/nsh' diff --git a/tests/fuzz/test_campaign.py b/tests/fuzz/test_campaign.py new file mode 100644 index 0000000..5701630 --- /dev/null +++ b/tests/fuzz/test_campaign.py @@ -0,0 +1,158 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import pytest + +from ntfc.fuzz.campaign import ( + CampaignError, + board_config_of, + build_target_conf, + load_fuzz_config, +) + +BUILD = "tests/fuzz/resources/campaign-build.yaml" +OSTEST = "tests/fuzz/resources/campaign-ostest.yaml" + + +def test_load_build_config(): + c = load_fuzz_config(BUILD) + assert c.feature == "build" + assert c.surface["scope"] == ["can"] + assert c.arch == "stm32" + assert c.mock is True + assert c.board == "sim:ntfc" + assert c.tree == "./external" + assert c.workers == 2 + assert c.needs_target is False + + +def test_load_ostest_config_needs_target(): + c = load_fuzz_config(OSTEST) + assert c.feature == "ostest" + assert c.needs_target is True + # ostest carries no build target -- it comes from the NTFC target config + assert c.board is None + assert c.tree is None + + +def test_defaults_applied(tmp_path): + p = tmp_path / "c.yaml" + p.write_text( + "feature: build\nboard: sim:nsh\ntree: .\nsurface: {symbols: [CAN]}\n" + ) + c = load_fuzz_config(str(p)) + assert c.strategy["mode"] == "single" + assert c.mock is False + assert c.timeout == 90 + assert c.parallel is False + assert c.require == [] + assert c.build_dir == "./build" + + +def test_missing_feature_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("surface: {symbols: [X]}\n") + with pytest.raises(CampaignError, match="feature"): + load_fuzz_config(str(p)) + + +def test_bad_feature_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("feature: nonsense\nsurface: {symbols: [X]}\n") + with pytest.raises(CampaignError, match="feature"): + load_fuzz_config(str(p)) + + +def test_missing_surface_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("feature: ostest\nsurface: {}\n") + with pytest.raises(CampaignError, match="surface"): + load_fuzz_config(str(p)) + + +def test_build_without_board_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("feature: build\ntree: .\nsurface: {symbols: [X]}\n") + with pytest.raises(CampaignError, match="board"): + load_fuzz_config(str(p)) + + +def test_build_without_tree_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("feature: mem\nboard: sim:nsh\nsurface: {symbols: [X]}\n") + with pytest.raises(CampaignError, match="tree"): + load_fuzz_config(str(p)) + + +def test_file_not_found_raises(tmp_path): + with pytest.raises(CampaignError, match="not found"): + load_fuzz_config(str(tmp_path / "nope.yaml")) + + +def test_not_a_mapping_raises(tmp_path): + p = tmp_path / "bad.yaml" + p.write_text("- a\n- b\n") + with pytest.raises(CampaignError, match="mapping"): + load_fuzz_config(str(p)) + + +def test_board_config_of_target(): + target = { + "product": { + "cores": {"c0": {"defconfig": "boards/sim/sim/sim/configs/nsh"}} + } + } + assert board_config_of(target) == "sim:nsh" + + +def test_board_config_of_skips_nonproduct_and_defconfigless(): + target = { + "config": {"cwd": "x"}, + "product": { + "cores": { + "c0": {"device": "sim"}, + "c1": {"defconfig": "boards/sim/sim/sim/configs/nsh"}, + } + }, + } + assert board_config_of(target) == "sim:nsh" + + +def test_board_config_of_without_configs_segment(): + target = { + "product": { + "cores": { + "c0": {"defconfig": "weird/path"}, + "c1": {"defconfig": "boards/sim/sim/sim/configs/nsh"}, + } + } + } + assert board_config_of(target) == "sim:nsh" + + +def test_board_config_of_missing_raises(): + with pytest.raises(CampaignError, match="defconfig"): + board_config_of({"product": {"cores": {"c0": {"device": "sim"}}}}) + + +def test_build_target_conf_shape(): + conf = build_target_conf("/x/tree", "/x/build") + assert conf["config"]["cwd"] == "/x/tree" + assert conf["config"]["build_dir"] == "/x/build" diff --git a/tests/fuzz/test_candidate.py b/tests/fuzz/test_candidate.py new file mode 100644 index 0000000..3ffe4c5 --- /dev/null +++ b/tests/fuzz/test_candidate.py @@ -0,0 +1,101 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from ntfc.fuzz import dataload +from ntfc.fuzz.candidate import ( + CRASH, + PASS, + TEST_FAIL, + TIMEOUT, + Outcome, + classify_run, +) + + +def _pats(): + return dataload.load_patterns() + + +def test_classify_pass(): + status, detail = classify_run( + "ostest_main: Exiting with status 0", 0, False, _pats() + ) + assert status == PASS + assert detail == "" + + +def test_classify_test_fail(): + status, detail = classify_run( + "ostest_main: Exiting with status 3", 0, False, _pats() + ) + assert status == TEST_FAIL + assert "3" in detail + + +def test_classify_timeout(): + status, _ = classify_run("partial output", -1, True, _pats()) + assert status == TIMEOUT + + +def test_classify_crash_on_assert(): + status, _ = classify_run("up_assert: blah", 0, False, _pats()) + assert status == CRASH + + +def test_classify_crash_on_segfault(): + status, detail = classify_run("nothing useful", -11, False, _pats()) + assert status == CRASH + assert "SIGSEGV" in detail + + +def test_classify_crash_when_no_completion(): + status, detail = classify_run("just booted", 0, False, _pats()) + assert status == CRASH + assert "did not finish" in detail + + +def test_outcome_ok_property(): + assert Outcome(frozenset({"CONFIG_A"}), PASS).ok is True + assert Outcome(frozenset({"CONFIG_A"}), CRASH).ok is False + + +def test_extract_errors_picks_error_lines(): + from ntfc.fuzz.candidate import extract_errors + + pats = _pats() + log = ( + "compiling foo.c\n" + "foo.c:12:3: error: 'GPIO_CAN2_RX' undeclared\n" + "some noise\n" + "ninja: error: build stopped\n" + ) + out = extract_errors(log, pats) + assert "GPIO_CAN2_RX" in out + assert "ninja: error" in out + assert "some noise" not in out + + +def test_extract_errors_falls_back_to_tail(): + from ntfc.fuzz.candidate import extract_errors + + pats = _pats() + log = "line1\nline2\nline3\n" # no error-pattern matches + out = extract_errors(log, pats, max_lines=2) + assert out == "line2\nline3" diff --git a/tests/fuzz/test_cmd_fuzz.py b/tests/fuzz/test_cmd_fuzz.py new file mode 100644 index 0000000..011618a --- /dev/null +++ b/tests/fuzz/test_cmd_fuzz.py @@ -0,0 +1,65 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from unittest.mock import patch + +from click.testing import CliRunner + +from ntfc.cli.main import main + +CAMP = "tests/fuzz/resources/campaign-build.yaml" +TARGET = "tests/fuzz/resources/target-sim.yaml" + + +def test_fuzz_command_invokes_engine(): + runner = CliRunner() + with patch("ntfc.cli.main.fuzz_run", return_value=0) as fr: + result = runner.invoke(main, ["fuzz", "--campaign", CAMP]) + assert result.exit_code == 0 + assert fr.called + ctx = fr.call_args[0][0] + assert ctx.runfuzz is True + assert ctx.fuzzpath == CAMP + + +def test_fuzz_command_nonzero_exits(): + runner = CliRunner() + with patch("ntfc.cli.main.fuzz_run", return_value=1): + result = runner.invoke(main, ["fuzz", "--campaign", CAMP]) + assert result.exit_code == 1 + + +def test_fuzz_command_passes_confpath_and_list(): + runner = CliRunner() + with patch("ntfc.cli.main.fuzz_run", return_value=0) as fr: + result = runner.invoke( + main, + ["fuzz", "--campaign", CAMP, "--list", "--confpath", TARGET], + ) + assert result.exit_code == 0 + ctx = fr.call_args[0][0] + assert ctx.fuzz_list is True + assert ctx.fuzz_confpath == TARGET + + +def test_fuzz_command_requires_campaign(): + runner = CliRunner() + result = runner.invoke(main, ["fuzz"]) + assert result.exit_code != 0 diff --git a/tests/fuzz/test_dataload.py b/tests/fuzz/test_dataload.py new file mode 100644 index 0000000..882657a --- /dev/null +++ b/tests/fuzz/test_dataload.py @@ -0,0 +1,83 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import re + +import pytest + +from ntfc.fuzz import dataload +from ntfc.fuzz.dataload import FuzzDataError + + +def test_load_profiles_stm32(): + profiles = dataload.load_profiles() + assert "stm32" in profiles + p = profiles["stm32"] + assert p.name == "stm32" + assert p.symbol_path_re.search("arch/arm/src/stm32h7/Kconfig") + assert p.symbol_path_re.search("arch/arm/src/common/stm32/Kconfig") + assert not p.symbol_path_re.search("net/Kconfig") + assert p.skip_prefixes == ( + "STM32_HAVE", + "ARCH_CHIP", + "STM32_FLASH_CONFIG_", + ) + assert "GPIO_" in p.mockable_prefixes + assert p.mockable_suffixes == ("_CLKIN", "_FREQUENCY") + + +def test_board_required_merges_generic_and_extra(): + p = dataload.get_profile("stm32") + br = p.board_required_re() + assert br.search("BOARD_LTDC_WIDTH must be defined") # generic + assert br.search("selected HSI48 as USB clock") # stm32 extra + + +def test_get_profile_unknown_raises_listing_known(): + with pytest.raises(FuzzDataError) as exc: + dataload.get_profile("does-not-exist") + assert "stm32" in str(exc.value) + + +def test_load_scopes_has_net_and_kernel(): + scopes = dataload.load_scopes() + assert re.compile(scopes["net"]).search("net/Kconfig") + assert re.compile(scopes["kernel"]).search("sched/Kconfig") + + +def test_load_patterns_keys(): + pats = dataload.load_patterns() + assert re.compile(pats["ostest_exit"]).search( + "ostest_main: Exiting with status 0" + ) + assert isinstance(pats["board_required"], list) + + +def test_missing_data_file_raises(tmp_path): + with pytest.raises(FuzzDataError) as exc: + dataload.load_scopes(data_dir=tmp_path) + assert "not found" in str(exc.value) + + +def test_patterns_missing_key_raises(tmp_path): + (tmp_path / "patterns.yaml").write_text("crash: x\n") + with pytest.raises(FuzzDataError) as exc: + dataload.load_patterns(data_dir=tmp_path) + assert "missing key" in str(exc.value) diff --git a/tests/fuzz/test_discover.py b/tests/fuzz/test_discover.py new file mode 100644 index 0000000..063548e --- /dev/null +++ b/tests/fuzz/test_discover.py @@ -0,0 +1,96 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from types import SimpleNamespace + +from ntfc.fuzz import dataload, discover +from ntfc.fuzz.discover import discover_fuzzable + + +def _sym( + name, filename, value="n", assignable=(0, 2), choice=None, prompt=True +): + node = SimpleNamespace(filename=filename, prompt=prompt) + return SimpleNamespace( + name=name, + type=discover.BOOL, + str_value=value, + assignable=assignable, + choice=choice, + nodes=[node], + ) + + +def _nonbool_sym(name, filename): + node = SimpleNamespace(filename=filename, prompt=True) + return SimpleNamespace( + name=name, + type=object(), # not discover.BOOL + str_value="n", + assignable=(0, 2), + choice=None, + nodes=[node], + ) + + +class _FakeKconf: + def __init__(self, syms): + self.unique_defined_syms = syms + + +def test_discovers_settable_arch_bool(): + p = dataload.get_profile("stm32") + sel = p.symbol_path_re + syms = [ + _sym("STM32_ADC2", "arch/arm/src/stm32h7/Kconfig"), # keep + _sym("STM32_HAVE_ADC2", "arch/arm/src/stm32h7/Kconfig"), # skip prefix + _sym("NET_TCP", "net/Kconfig"), # out of scope + _sym( + "STM32_ADC1", "arch/arm/src/stm32h7/Kconfig", value="y" + ), # already y + _sym( + "STM32_ADC3", "arch/arm/src/stm32h7/Kconfig", assignable=(0,) + ), # not settable + _sym( + "STM32_ADC4", "arch/arm/src/stm32h7/Kconfig", prompt=False + ), # no prompt + _nonbool_sym( + "STM32_UART5_RXDMA", "arch/arm/src/stm32h7/Kconfig" + ), # not a bool + ] + out = discover_fuzzable(_FakeKconf(syms), profile=p, selector=sel) + assert [s.name for s in out] == ["STM32_ADC2"] + assert out[0].target == "y" + assert out[0].kind == "bool" + + +def test_include_choices_toggle(): + p = dataload.get_profile("stm32") + sel = p.symbol_path_re + choice = object() + syms = [ + _sym( + "STM32_CHOICEMEMBER", "arch/arm/src/stm32h7/Kconfig", choice=choice + ) + ] + assert discover_fuzzable(_FakeKconf(syms), p, sel) == [] + got = discover_fuzzable(_FakeKconf(syms), p, sel, include_choices=True) + assert [s.name for s in got] == ["STM32_CHOICEMEMBER"] + assert got[0].in_choice is True diff --git a/tests/fuzz/test_engine_build.py b/tests/fuzz/test_engine_build.py new file mode 100644 index 0000000..8ffb04a --- /dev/null +++ b/tests/fuzz/test_engine_build.py @@ -0,0 +1,105 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from ntfc.fuzz.campaign import load_fuzz_config +from ntfc.fuzz.candidate import BUILD_FAIL, PASS, Outcome +from ntfc.fuzz.engine import run_build, run_campaign +from ntfc.fuzz.model import SymbolInfo + +BUILD = "tests/fuzz/resources/campaign-build.yaml" + + +def _fake_discover(*args, **kwargs): + return [ + SymbolInfo("CAN1", "bool", "n", "y", ["drivers/can/Kconfig"], False), + SymbolInfo("CAN2", "bool", "n", "y", ["drivers/can/Kconfig"], False), + ] + + +def _fake_build(builder, board_config, configs, build_dir, **kwargs): + if "CONFIG_CAN2" in configs: + return Outcome( + configs, + BUILD_FAIL, + "error: 'GPIO_CAN2_RX' undeclared", + log_path="/tmp/build-CONFIG_CAN2.log", + ) + return Outcome(configs, PASS) + + +def test_run_build_collects_outcomes(): + c = load_fuzz_config(BUILD) + report = run_build(c, discover_fn=_fake_discover, build_fn=_fake_build) + data = report.json() + statuses = {o["status"] for o in data["outcomes"]} + assert BUILD_FAIL in statuses + assert PASS in statuses + assert report.base == "sim:ntfc" + fails = [o for o in data["outcomes"] if o["status"] == BUILD_FAIL] + assert any("CONFIG_CAN2" in o["configs"] for o in fails) + assert report.failed() is True + + +def test_run_build_symbols_surface_skips_discovery(tmp_path): + p = tmp_path / "c.yaml" + p.write_text( + "feature: build\nboard: sim:nsh\ntree: .\n" + "surface: {symbols: [CAN1, CAN2]}\n" + ) + c = load_fuzz_config(str(p)) + + def boom_discover(*a, **k): + raise AssertionError("discovery must not run for symbols surface") + + report = run_build(c, discover_fn=boom_discover, build_fn=_fake_build) + assert len(report.outcomes) == 2 + + +def test_run_campaign_dispatches_build(): + c = load_fuzz_config(BUILD) + report = run_campaign( + c, None, discover_fn=_fake_discover, build_fn=_fake_build + ) + assert report.feature == "build" + assert len(report.outcomes) == 2 + + +def test_run_build_sequential(): + c = load_fuzz_config(BUILD) + c.parallel = False + report = run_build(c, discover_fn=_fake_discover, build_fn=_fake_build) + assert len(report.outcomes) == 2 + + +def test_run_build_emits_progress(): + c = load_fuzz_config(BUILD) + c.parallel = False + msgs = [] + run_build( + c, + discover_fn=_fake_discover, + build_fn=_fake_build, + progress=msgs.append, + ) + joined = "\n".join(msgs) + assert "build sweep: 2 candidate(s)" in joined + assert any("build-fail" in m and "GPIO_CAN2_RX" in m for m in msgs) + assert any("log saved: /tmp/build-CONFIG_CAN2.log" in m for m in msgs) + assert "build sweep complete" in joined diff --git a/tests/fuzz/test_engine_helpers.py b/tests/fuzz/test_engine_helpers.py new file mode 100644 index 0000000..6c4b767 --- /dev/null +++ b/tests/fuzz/test_engine_helpers.py @@ -0,0 +1,134 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import os +from pathlib import Path +from types import SimpleNamespace + +from ntfc.fuzz.campaign import load_fuzz_config +from ntfc.fuzz.candidate import PASS, Outcome +from ntfc.fuzz.engine import ( + _build_root, + _features_from_surface, + _subsets, + _surface_names, + plan_candidates, + run_mem, +) + +BUILD = "tests/fuzz/resources/campaign-build.yaml" +MEM = "tests/fuzz/resources/campaign-mem.yaml" +OSTEST = "tests/fuzz/resources/campaign-ostest.yaml" + + +def test_subsets_single(): + assert _subsets({"mode": "single"}, ["A", "B"]) == [ + frozenset({"A"}), + frozenset({"B"}), + ] + + +def test_subsets_random_is_seeded(): + a = _subsets( + {"mode": "random", "rounds": 4, "size": 2, "seed": 7}, ["A", "B", "C"] + ) + assert len(a) == 4 + + +def test_subsets_systematic(): + subs = _subsets({"mode": "marginal"}, ["A", "B"]) + assert frozenset({"A", "B"}) in subs + + +def test_build_root_is_under_build_dir(): + # bound at import time, before the conftest fixture patches it out + assert _build_root("bd") == os.path.join("bd", "fuzz") + + +def test_subsets_limit_caps_candidates(): + subs = _subsets({"mode": "single", "limit": 2}, ["A", "B", "C"]) + assert subs == [frozenset({"A"}), frozenset({"B"})] + + +def test_plan_candidates_build_sweeps_discovered_surface(): + fuzz = load_fuzz_config(BUILD) + fuzz.strategy["limit"] = 1 + + def fake_discover(root, board, profile, selector, include_choices=False): + return [ + SimpleNamespace(name="STM32_ADC1"), + SimpleNamespace(name="STM32_ADC2"), + ] + + subs = plan_candidates(fuzz, discover_fn=fake_discover) + assert subs == [frozenset({"CONFIG_STM32_ADC1"})] + + +def test_plan_candidates_mem_starts_with_baseline(): + subs = plan_candidates(load_fuzz_config(MEM)) + assert subs[0] == frozenset() + assert frozenset({"CONFIG_FS_FAT"}) in subs + assert frozenset({"CONFIG_FS_FAT", "CONFIG_CRYPTO"}) in subs + + +def test_plan_candidates_ostest_baseline_and_skip(): + fuzz = load_fuzz_config(OSTEST) + subs = plan_candidates(fuzz) + assert subs[0] == frozenset() + assert len(subs) == 7 # baseline + 6 random rounds + + fuzz.strategy["skip_baseline"] = True + subs = plan_candidates(fuzz) + assert len(subs) == 6 + assert frozenset() not in subs + + +def test_surface_names_features_flattens(): + fuzz = load_fuzz_config(BUILD) + fuzz.surface = {"features": {"net": ["CONFIG_NET", "CONFIG_NETDEV"]}} + names = _surface_names(fuzz, Path("."), None, "sim:nsh", None) + assert set(names) == {"CONFIG_NET", "CONFIG_NETDEV"} + + +def test_features_from_surface_group(): + feats, req = _features_from_surface( + {"features": {"net": ["CONFIG_NET", "CONFIG_NETDEV"]}}, + ["CONFIG_SCHED_HPWORK"], + ) + assert feats[0].name == "net" + assert req == frozenset({"CONFIG_SCHED_HPWORK"}) + + +def test_mem_max_builds_cap(): + c = load_fuzz_config(MEM) + c.strategy["mode"] = "full" + c.strategy["max_builds"] = 1 + + def build(builder, board_config, configs, build_dir, **kw): + return Outcome( + frozenset(configs), PASS, elf_path=os.path.join(build_dir, "nuttx") + ) + + report = run_mem( + c, + build_fn=build, + size_fn=lambda e, size_tool="size": (10000, 200, 500), + ) + assert report.memory == [] diff --git a/tests/fuzz/test_engine_mem.py b/tests/fuzz/test_engine_mem.py new file mode 100644 index 0000000..b584579 --- /dev/null +++ b/tests/fuzz/test_engine_mem.py @@ -0,0 +1,103 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import os + +from ntfc.fuzz.campaign import load_fuzz_config +from ntfc.fuzz.candidate import BUILD_FAIL, PASS, Outcome +from ntfc.fuzz.engine import run_campaign, run_mem + +MEM = "tests/fuzz/resources/campaign-mem.yaml" + + +def test_mem_reports_positive_delta(): + c = load_fuzz_config(MEM) + built = {} + + def fake_build(builder, board_config, configs, build_dir, **kw): + built[build_dir] = frozenset(configs) + return Outcome( + frozenset(configs), PASS, elf_path=os.path.join(build_dir, "nuttx") + ) + + def fake_size(elf_path, size_tool="size"): + cfgs = built.get(os.path.dirname(elf_path), frozenset()) + extra = 2000 if "CONFIG_FS_FAT" in cfgs else 0 + return (10000 + extra, 200, 500) + + report = run_mem(c, build_fn=fake_build, size_fn=fake_size) + table = report.json()["memory"] + fat = [r for r in table if r["feature"] == "CONFIG_FS_FAT"][0] + assert fat["flash_delta"] == 2000 + assert fat["ram_delta"] == 0 + + +def test_mem_baseline_build_fail_returns_empty(): + c = load_fuzz_config(MEM) + + def failing_build(builder, board_config, configs, build_dir, **kw): + return Outcome(frozenset(configs), BUILD_FAIL, "boom") + + report = run_mem(c, build_fn=failing_build) + assert report.memory == [] + assert report.baseline is not None + assert report.baseline.status == BUILD_FAIL + + +def test_mem_variant_build_fail_skipped(): + c = load_fuzz_config(MEM) + + def build(builder, board_config, configs, build_dir, **kw): + if "CONFIG_FS_FAT" in configs: + return Outcome(frozenset(configs), BUILD_FAIL, "boom") + return Outcome( + frozenset(configs), PASS, elf_path=os.path.join(build_dir, "nuttx") + ) + + def size(elf_path, size_tool="size"): + return (10000, 200, 500) + + report = run_mem(c, build_fn=build, size_fn=size) + feats = {r["feature"] for r in report.memory} + assert "CONFIG_FS_FAT" not in feats + assert "CONFIG_CRYPTO" in feats + # failed variants ({FS_FAT} and the all-on pair) are recorded in the + # report, not silently dropped + assert report.counts()[BUILD_FAIL] == 2 + assert report.failed() is True + assert report.baseline is not None + assert report.baseline.status == PASS + + +def test_run_campaign_dispatches_mem(): + c = load_fuzz_config(MEM) + + def build(builder, board_config, configs, build_dir, **kw): + return Outcome( + frozenset(configs), PASS, elf_path=os.path.join(build_dir, "nuttx") + ) + + report = run_campaign( + c, + None, + build_fn=build, + size_fn=lambda e, size_tool="size": (10000, 200, 500), + ) + assert report.feature == "mem" diff --git a/tests/fuzz/test_engine_ostest.py b/tests/fuzz/test_engine_ostest.py new file mode 100644 index 0000000..cc3fb6d --- /dev/null +++ b/tests/fuzz/test_engine_ostest.py @@ -0,0 +1,140 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import pytest +import yaml + +from ntfc.fuzz.campaign import CampaignError, load_fuzz_config +from ntfc.fuzz.candidate import BUILD_FAIL, CRASH, PASS, Outcome +from ntfc.fuzz.engine import run_campaign, run_ostest + +OSTEST = "tests/fuzz/resources/campaign-ostest.yaml" +TARGET = "tests/fuzz/resources/target-sim.yaml" + + +def _target(): + with open(TARGET, encoding="utf-8") as f: + return yaml.safe_load(f) + + +def _pass_build(builder, board_config, configs, build_dir, **kw): + return Outcome(configs, PASS, elf_path=build_dir + "/nuttx") + + +def _crash_if_bad_run(core_conf, configs, patterns, timeout, **kwargs): + if "CONFIG_BAD" in configs: + return Outcome(configs, CRASH, "assert") + return Outcome(configs, PASS) + + +def test_ostest_sweep_minimizes_to_culprit(): + c = load_fuzz_config(OSTEST) + report = run_ostest( + c, _target(), build_fn=_pass_build, run_fn=_crash_if_bad_run + ) + data = report.json() + assert data["baseline"]["status"] == PASS + assert data["base"] == "sim:nsh" + mins = data["minimized"] + assert mins + assert all("CONFIG_BAD" in m["minimal"] for m in mins) + + +def test_ostest_build_fail_short_circuits_run(): + c = load_fuzz_config(OSTEST) + c.strategy["minimize"] = False + c.strategy["skip_baseline"] = True + + def failing_build(builder, board_config, configs, build_dir, **kw): + return Outcome(configs, BUILD_FAIL, "boom") + + def must_not_run(*a, **k): + raise AssertionError("run must not happen when build fails") + + report = run_ostest( + c, _target(), build_fn=failing_build, run_fn=must_not_run + ) + assert all(o.status == BUILD_FAIL for o in report.outcomes) + + +def test_ostest_baseline_build_fail_aborts_sweep(): + c = load_fuzz_config(OSTEST) + + def failing_build(builder, board_config, configs, build_dir, **kw): + return Outcome(configs, BUILD_FAIL, "no nuttx tree") + + def must_not_run(*a, **k): + raise AssertionError("nothing should run if the baseline won't build") + + report = run_ostest( + c, _target(), build_fn=failing_build, run_fn=must_not_run + ) + assert report.baseline.status == BUILD_FAIL + assert report.outcomes == [] + + +def test_ostest_build_fails_are_not_minimized(): + c = load_fuzz_config(OSTEST) + c.strategy["skip_baseline"] = True + c.strategy["minimize"] = True + + def build(builder, board_config, configs, build_dir, **kw): + if len(configs) > 1: + return Outcome(configs, BUILD_FAIL, "boom") + return Outcome(configs, PASS, elf_path=build_dir + "/nuttx") + + def run(core_conf, configs, patterns, timeout, **kwargs): + return Outcome(configs, PASS) + + report = run_ostest(c, _target(), build_fn=build, run_fn=run) + assert report.minimized == [] + + +def test_ostest_skip_baseline(): + c = load_fuzz_config(OSTEST) + c.strategy["skip_baseline"] = True + c.strategy["minimize"] = False + report = run_ostest( + c, _target(), build_fn=_pass_build, run_fn=_crash_if_bad_run + ) + assert report.baseline is None + + +def test_ostest_scope_surface_rejected(tmp_path): + p = tmp_path / "c.yaml" + p.write_text("feature: ostest\nsurface: {scope: [net]}\n") + c = load_fuzz_config(str(p)) + with pytest.raises(CampaignError, match="symbols"): + run_ostest(c, _target(), build_fn=_pass_build) + + +def test_run_campaign_dispatches_ostest(): + c = load_fuzz_config(OSTEST) + c.strategy["minimize"] = False + report = run_campaign( + c, _target(), build_fn=_pass_build, run_fn=_crash_if_bad_run + ) + assert report.feature == "ostest" + + +def test_run_campaign_ostest_without_target_raises(): + c = load_fuzz_config(OSTEST) + with pytest.raises(CampaignError, match="target"): + run_campaign(c, None) diff --git a/tests/fuzz/test_features.py b/tests/fuzz/test_features.py new file mode 100644 index 0000000..39588ce --- /dev/null +++ b/tests/fuzz/test_features.py @@ -0,0 +1,88 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import dataclasses + +import pytest + +from ntfc.fuzz.features import Feature, normalize, parse_features + + +def test_options_each_its_own_feature(): + feats, req = parse_features("CONFIG_FS_FAT,CONFIG_CRYPTO", (), None) + names = {f.name for f in feats} + assert names == {"CONFIG_FS_FAT", "CONFIG_CRYPTO"} + assert req == frozenset() + + +def test_options_configs_carry_prefix(): + feats, _ = parse_features("FS_FAT", (), None) + assert feats[0].configs == frozenset({"CONFIG_FS_FAT"}) + + +def test_named_feature_group(): + feats, req = parse_features( + None, ("net=CONFIG_NET,CONFIG_NETDEV_LATEINIT",), None + ) + assert feats[0].name == "net" + assert feats[0].configs == frozenset( + {"CONFIG_NET", "CONFIG_NETDEV_LATEINIT"} + ) + + +def test_require_parsed(): + _, req = parse_features("CONFIG_A", (), "CONFIG_SCHED_HPWORK") + assert req == frozenset({"CONFIG_SCHED_HPWORK"}) + + +def test_duplicate_feature_raises(): + # an --options entry and a --feature group resolving to the same name + with pytest.raises(ValueError): + parse_features("CONFIG_A", ("CONFIG_A=CONFIG_B",), None) + + +def test_empty_raises(): + with pytest.raises(ValueError): + parse_features(None, (), None) + + +def test_normalize_strips_prefix_and_dedups(): + assert normalize(["CONFIG_A", "A", " B "]) == ["A", "B"] + + +def test_normalize_skips_empty_tokens(): + assert normalize(["", " ", "A"]) == ["A"] + + +def test_feature_spec_without_name_uses_first_config(): + feats, _ = parse_features(None, ("CONFIG_NET,CONFIG_NETDEV",), None) + assert feats[0].name == "CONFIG_NET" + assert feats[0].configs == frozenset({"CONFIG_NET", "CONFIG_NETDEV"}) + + +def test_empty_feature_group_raises(): + with pytest.raises(ValueError): + parse_features(None, ("=",), None) + + +def test_feature_is_frozen(): + f = Feature("x", frozenset({"CONFIG_X"})) + with pytest.raises(dataclasses.FrozenInstanceError): + f.name = "y" # type: ignore[misc] diff --git a/tests/fuzz/test_memsize.py b/tests/fuzz/test_memsize.py new file mode 100644 index 0000000..d2cc54d --- /dev/null +++ b/tests/fuzz/test_memsize.py @@ -0,0 +1,66 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from unittest.mock import MagicMock, patch + +from ntfc.fuzz import memsize + + +def test_elf_size_parses_size_output(): + out = ( + " text\t data\t bss\t dec\t hex\tfilename\n" + " 10000\t 200\t 500\t 10700\t 29cc\tnuttx\n" + ) + with patch( + "subprocess.run", return_value=MagicMock(stdout=out, returncode=0) + ): + assert memsize.elf_size("nuttx") == (10000, 200, 500) + + +def test_elf_size_returns_zeros_on_failure(): + with patch( + "subprocess.run", return_value=MagicMock(stdout="", returncode=1) + ): + assert memsize.elf_size("nuttx") == (0, 0, 0) + + +def test_elf_size_returns_zeros_on_oserror(): + with patch("subprocess.run", side_effect=OSError): + assert memsize.elf_size("nuttx") == (0, 0, 0) + + +def test_elf_size_returns_zeros_on_short_output(): + with patch( + "subprocess.run", + return_value=MagicMock(stdout="only a header line\n", returncode=0), + ): + assert memsize.elf_size("nuttx") == (0, 0, 0) + + +def test_elf_size_returns_zeros_on_unparseable_row(): + out = "text data bss\nnot numbers here really\n" + with patch( + "subprocess.run", return_value=MagicMock(stdout=out, returncode=0) + ): + assert memsize.elf_size("nuttx") == (0, 0, 0) + + +def test_footprint(): + assert memsize.footprint(10000, 200, 500) == (10200, 700) diff --git a/tests/fuzz/test_mocks.py b/tests/fuzz/test_mocks.py new file mode 100644 index 0000000..8cb1bff --- /dev/null +++ b/tests/fuzz/test_mocks.py @@ -0,0 +1,80 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from pathlib import Path + +from ntfc.fuzz import dataload +from ntfc.fuzz.mocks import append_mock_defines, missing_board_constants + + +def test_mocks_gpio_pin_with_suggestion(): + p = dataload.get_profile("stm32") + log = ( + "error: 'GPIO_CAN2_RX' undeclared (first use in this function); " + "did you mean 'GPIO_CAN2_RX_1'?" + ) + out = missing_board_constants(log, p) + assert out.get("GPIO_CAN2_RX") == "GPIO_CAN2_RX_1" + + +def test_mocks_undeclared_without_suggestion_gets_zero(): + p = dataload.get_profile("stm32") + log = "error: 'BOARD_LTDC_HEIGHT' undeclared here (not in a function)" + assert missing_board_constants(log, p) == {"BOARD_LTDC_HEIGHT": "0"} + + +def test_mocks_must_be_defined_gets_one(): + p = dataload.get_profile("stm32") + log = "#error BOARD_LTDC_WIDTH must be defined in the board.h header file" + assert missing_board_constants(log, p) == {"BOARD_LTDC_WIDTH": "1"} + + +def test_mocks_non_board_identifier_ignored(): + p = dataload.get_profile("stm32") + log = "error: 'some_local_var' undeclared" + assert missing_board_constants(log, p) == {} + + +def test_mocks_suffix_clkin_is_mockable(): + p = dataload.get_profile("stm32") + log = "error: 'STM32_TIM6_CLKIN' undeclared" + assert missing_board_constants(log, p) == {"STM32_TIM6_CLKIN": "0"} + + +def test_mocks_must_be_defined_non_board_ignored(): + p = dataload.get_profile("stm32") + log = "#error SOMETHING must be defined in the header file" + assert missing_board_constants(log, p) == {} + + +def test_append_mock_defines_writes_block(tmp_path): + cfg = Path(tmp_path) / "config.h" + cfg.write_text("#define CONFIG_FOO 1\n") + append_mock_defines(cfg, {"GPIO_CAN2_RX": "GPIO_CAN2_RX_1"}) + text = cfg.read_text() + assert "#define GPIO_CAN2_RX GPIO_CAN2_RX_1" in text + assert "#define CONFIG_FOO 1" in text # original preserved + + +def test_append_mock_defines_empty_is_noop(tmp_path): + cfg = Path(tmp_path) / "config.h" + cfg.write_text("#define CONFIG_FOO 1\n") + append_mock_defines(cfg, {}) + assert cfg.read_text() == "#define CONFIG_FOO 1\n" diff --git a/tests/fuzz/test_model.py b/tests/fuzz/test_model.py new file mode 100644 index 0000000..afde223 --- /dev/null +++ b/tests/fuzz/test_model.py @@ -0,0 +1,30 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from ntfc.fuzz.model import SymbolInfo + + +def test_symbolinfo_to_dict(): + s = SymbolInfo("STM32_ADC2", "bool", "n", "y", ["a/Kconfig"], False) + d = s.to_dict() + assert d["name"] == "STM32_ADC2" + assert d["kind"] == "bool" + assert d["files"] == ["a/Kconfig"] + assert d["in_choice"] is False diff --git a/tests/fuzz/test_packaged_data.py b/tests/fuzz/test_packaged_data.py new file mode 100644 index 0000000..4738bf4 --- /dev/null +++ b/tests/fuzz/test_packaged_data.py @@ -0,0 +1,28 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from ntfc.fuzz import dataload + + +def test_packaged_data_dir_resolves(): + assert (dataload.DATA_DIR / "arch-profiles.yaml").exists() + assert (dataload.DATA_DIR / "scopes.yaml").exists() + assert (dataload.DATA_DIR / "patterns.yaml").exists() + assert "stm32" in dataload.load_profiles() diff --git a/tests/fuzz/test_profiles.py b/tests/fuzz/test_profiles.py new file mode 100644 index 0000000..82d5349 --- /dev/null +++ b/tests/fuzz/test_profiles.py @@ -0,0 +1,80 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +from types import SimpleNamespace + +import pytest + +from ntfc.fuzz import dataload +from ntfc.fuzz.dataload import FuzzDataError +from ntfc.fuzz.profiles import build_selector + + +def _fixture(): + return dataload.get_profile("stm32"), dataload.load_scopes() + + +def test_selector_none_is_arch_surface(): + p, scopes = _fixture() + sel = build_selector(p, None, scopes) + assert sel.search("arch/arm/src/stm32h7/Kconfig") + + +def test_selector_named_scopes(): + p, scopes = _fixture() + sel = build_selector(p, "net,fs", scopes) + assert sel.search("net/Kconfig") + assert sel.search("fs/vfs/Kconfig") + assert not sel.search("arch/arm/src/stm32h7/Kconfig") + + +def test_selector_all_matches_everything(): + p, scopes = _fixture() + sel = build_selector(p, "all", scopes) + assert sel.search("anything/at/all") + + +def test_selector_arch_scope(): + p, scopes = _fixture() + sel = build_selector(p, "arch", scopes) + assert sel.search("arch/arm/src/stm32h7/Kconfig") + assert not sel.search("net/Kconfig") + + +def test_selector_unknown_raises(): + p, scopes = _fixture() + with pytest.raises(FuzzDataError): + build_selector(p, "nonsense", scopes) + + +def test_config_required_re(): + p = dataload.get_profile("stm32") + cr = p.config_required_re() + assert cr.search("feature requires CONFIG_NET") + + +def test_is_arch_symbol(): + p = dataload.get_profile("stm32") + in_arch = SimpleNamespace( + nodes=[SimpleNamespace(filename="arch/arm/src/stm32h7/Kconfig")] + ) + not_arch = SimpleNamespace(nodes=[SimpleNamespace(filename="net/Kconfig")]) + assert p.is_arch_symbol(in_arch) is True + assert p.is_arch_symbol(not_arch) is False diff --git a/tests/fuzz/test_report_write.py b/tests/fuzz/test_report_write.py new file mode 100644 index 0000000..4c1a0bd --- /dev/null +++ b/tests/fuzz/test_report_write.py @@ -0,0 +1,87 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import json +import os + +from ntfc.fuzz.candidate import BUILD_FAIL, PASS, PASS_MOCKED, Outcome +from ntfc.fuzz.report import FuzzReport + + +def test_write_emits_txt_and_json(tmp_path): + r = FuzzReport(base="sim:ntfc", feature="build") + r.add(Outcome(frozenset({"CONFIG_A"}), PASS)) + r.add(Outcome(frozenset({"CONFIG_B"}), BUILD_FAIL, "boom", log_path="x")) + r.write(str(tmp_path)) + + assert os.path.exists(tmp_path / "fuzz-report.txt") + with open(tmp_path / "fuzz-report.json", encoding="utf-8") as f: + data = json.load(f) + assert data["feature"] == "build" + assert len(data["outcomes"]) == 2 + assert data["counts"][PASS] == 1 + assert data["counts"][BUILD_FAIL] == 1 + + +def test_text_lists_failures_and_baseline(): + r = FuzzReport(base="sim:ntfc", feature="ostest") + r.baseline = Outcome(frozenset(), PASS) + r.add(Outcome(frozenset({"CONFIG_B"}), BUILD_FAIL, "boom")) + r.minimized.append( + ( + Outcome(frozenset({"CONFIG_B", "CONFIG_C"}), BUILD_FAIL), + frozenset({"CONFIG_B"}), + ) + ) + text = r.text() + assert "baseline: pass" in text + assert "failing candidates:" in text + assert "minimised failures" in text + + +def test_text_failure_without_detail(): + r = FuzzReport(base="sim:ntfc", feature="build") + r.add(Outcome(frozenset({"CONFIG_B"}), BUILD_FAIL)) # no detail, no log + text = r.text() + assert "CONFIG_B" in text + assert "failing candidates:" in text + + +def test_text_all_passed(): + r = FuzzReport(base="sim:ntfc", feature="build") + r.add(Outcome(frozenset({"CONFIG_A"}), PASS)) + assert "all candidates passed." in r.text() + assert r.failed() is False + + +def test_pass_mocked_counts_as_ok(): + r = FuzzReport(base="sim:ntfc", feature="build") + r.add(Outcome(frozenset({"CONFIG_A"}), PASS_MOCKED)) + assert r.failed() is False + + +def test_memory_rows_rendered(): + r = FuzzReport(base="sim:ntfc", feature="mem") + r.memory.append( + {"feature": "CONFIG_FS_FAT", "flash_delta": 2000, "ram_delta": 0} + ) + text = r.text() + assert "memory (flash/ram delta" in text + assert "CONFIG_FS_FAT" in text diff --git a/tests/fuzz/test_strategy.py b/tests/fuzz/test_strategy.py new file mode 100644 index 0000000..94ea1b9 --- /dev/null +++ b/tests/fuzz/test_strategy.py @@ -0,0 +1,85 @@ +############################################################################ +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +import pytest + +from ntfc.fuzz import strategy + + +def test_single_subsets(): + subs = strategy.single_subsets(["A", "B", "C"]) + assert subs == [frozenset({"A"}), frozenset({"B"}), frozenset({"C"})] + + +def test_random_subsets_deterministic_with_seed(): + a = strategy.random_subsets(["A", "B", "C", "D"], rounds=5, size=3, seed=1) + b = strategy.random_subsets(["A", "B", "C", "D"], rounds=5, size=3, seed=1) + assert a == b + assert len(a) == 5 + assert all(1 <= len(s) <= 3 for s in a) + + +def test_systematic_marginal(): + subs = strategy.systematic_subsets(["A", "B"], "marginal") + assert frozenset({"A"}) in subs + assert frozenset({"A", "B"}) in subs + + +def test_systematic_marginal_single_name_has_no_all_on(): + subs = strategy.systematic_subsets(["A"], "marginal") + assert subs == [frozenset({"A"})] + + +def test_systematic_pairs(): + subs = strategy.systematic_subsets(["A", "B", "C"], "pairs") + assert frozenset({"A", "B"}) in subs + assert frozenset({"A"}) in subs + + +def test_systematic_full_covers_all_nonempty_subsets(): + subs = strategy.systematic_subsets(["A", "B"], "full") + assert set(subs) == { + frozenset({"A"}), + frozenset({"B"}), + frozenset({"A", "B"}), + } + + +def test_systematic_unknown_mode_raises(): + with pytest.raises(ValueError): + strategy.systematic_subsets(["A"], "bogus") + + +def test_systematic_subsets_are_unique(): + subs = strategy.systematic_subsets(["A", "A"], "pairs") + assert len(subs) == len(set(subs)) + + +def test_ddmin_finds_single_culprit(): + minimal = strategy.ddmin(["A", "B", "C", "D"], lambda s: "C" in s) + assert minimal == ["C"] + + +def test_ddmin_irreducible_set_increases_granularity(): + # only the full set reproduces the failure: ddmin must increase + # granularity (no single-chunk complement fails) and return everything. + items = ["A", "B", "C", "D"] + minimal = strategy.ddmin(items, lambda s: set(s) == {"A", "B", "C", "D"}) + assert sorted(minimal) == items diff --git a/tests/test_builder.py b/tests/test_builder.py index 0ed6d65..93ce78c 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -19,6 +19,7 @@ ############################################################################ import copy +import os import tempfile from pathlib import Path from unittest.mock import patch @@ -331,6 +332,20 @@ def test_builder_apply_kconfig_overrides() -> None: assert "# CONFIG_OFF is not set\n" in cfg_text assert "CONFIG_KEEP=y\n" in cfg_text + # an identical re-apply must not rewrite the file (an mtime bump + # would force a config.h regeneration that wipes injected mocks) + mtime = os.stat(cfg_path).st_mtime_ns + b._apply_kconfig_overrides( + str(cfg_path), + { + "CONFIG_REPLACE": "newval", + "CONFIG_DISABLE_ME": "y", + "CONFIG_APPEND": "0x10", + "CONFIG_OFF": False, + }, + ) + assert os.stat(cfg_path).st_mtime_ns == mtime + missing_path = Path(tmpdir) / "missing.config" b._apply_kconfig_overrides(str(missing_path), {"CONFIG_X": "y"}) @@ -484,3 +499,139 @@ def test_builder_raises_when_cwd_missing() -> None: BuilderConfigError, match="not found cwd in YAML configuration" ): b.build_all() + + +def test_build_candidate_captures_failure(tmp_path): + from ntfc.builder import CandidateBuild + + conf = copy.deepcopy(conf_dir) + conf["config"]["cwd"] = str(tmp_path) + conf["config"]["build_dir"] = str(tmp_path / "build") + b = NuttXBuilder(conf) + + def fake_capture(cmd, env=None): + if "olddefconfig" in cmd: + return 0, "expanded" + if "--build" in cmd: + return 1, "ninja: error: boom" + return 0, "configured" + + with ( + patch.object(b, "_run_capture", side_effect=fake_capture), + patch.object(b, "_apply_kconfig_overrides"), + ): + res = b.build_candidate( + "sim:ntfc", + {"CONFIG_FOO": True}, + build_dir=str(tmp_path / "build" / "cand0"), + ) + + assert isinstance(res, CandidateBuild) + assert res.ok is False + assert "expanded" in res.log + assert "boom" in res.log + assert res.elf_path.endswith("/nuttx") + + +def test_build_candidate_no_configure_compiles_as_is(tmp_path): + conf = copy.deepcopy(conf_dir) + conf["config"]["cwd"] = str(tmp_path) + conf["config"]["build_dir"] = str(tmp_path / "build") + b = NuttXBuilder(conf) + + calls = [] + + def fake_capture(cmd, env=None): + calls.append(cmd) + return 0, "built" + + with ( + patch.object(b, "_run_capture", side_effect=fake_capture), + patch.object(b, "_apply_kconfig_overrides") as ovr, + ): + res = b.build_candidate( + "sim:ntfc", + {"CONFIG_FOO": True}, + build_dir=str(tmp_path / "build" / "cand4"), + configure=False, + ) + + assert res.ok is True + assert len(calls) == 1 # only the compile step + assert calls[0][:2] == ["cmake", "--build"] + ovr.assert_not_called() + + +def test_build_candidate_olddefconfig_failure_skips_build(tmp_path): + conf = copy.deepcopy(conf_dir) + conf["config"]["cwd"] = str(tmp_path) + conf["config"]["build_dir"] = str(tmp_path / "build") + b = NuttXBuilder(conf) + + calls = [] + + def fake_capture(cmd, env=None): + calls.append(cmd) + if "olddefconfig" in cmd: + return 1, "olddefconfig: boom" + return 0, "configured" + + with ( + patch.object(b, "_run_capture", side_effect=fake_capture), + patch.object(b, "_apply_kconfig_overrides"), + ): + res = b.build_candidate( + "sim:ntfc", + {"CONFIG_FOO": True}, + build_dir=str(tmp_path / "build" / "cand3"), + ) + + assert res.ok is False + assert "olddefconfig: boom" in res.log + assert len(calls) == 2 # configure + olddefconfig; build never reached + + +def test_build_candidate_configure_failure_skips_build(tmp_path): + conf = copy.deepcopy(conf_dir) + conf["config"]["cwd"] = str(tmp_path) + conf["config"]["build_dir"] = str(tmp_path / "build") + b = NuttXBuilder(conf) + + calls = [] + + def fake_capture(cmd, env=None): + calls.append(cmd) + return 1, "cmake configure failed" + + with ( + patch.object(b, "_run_capture", side_effect=fake_capture), + patch.object(b, "_apply_kconfig_overrides") as ovr, + ): + res = b.build_candidate( + "sim:ntfc", {}, build_dir=str(tmp_path / "build" / "cand2") + ) + + assert res.ok is False + assert len(calls) == 1 # build step never reached + ovr.assert_not_called() + + +def test_build_candidate_success(tmp_path): + conf = copy.deepcopy(conf_dir) + conf["config"]["cwd"] = str(tmp_path) + conf["config"]["build_dir"] = str(tmp_path / "build") + b = NuttXBuilder(conf) + + with ( + patch.object(b, "_run_capture", return_value=(0, "ok")), + patch.object(b, "_apply_kconfig_overrides"), + ): + res = b.build_candidate( + "sim:ntfc", + {}, + build_dir=str(tmp_path / "build" / "cand1"), + jobs=4, + ) + assert res.ok is True + assert res.elf_path.endswith("/nuttx") + assert res.conf_path.endswith("/.config")