From bbf1d39c852b4182fc111363bce5bdb80d7c6dc6 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Thu, 10 Sep 2026 17:41:56 -0400 Subject: [PATCH] fix(core): dominant-lang, manifest-scoped scanning, git quotepath -- closes #2555 Three defects observed in a single galaxyscope scan of expressjs/express: 1. Dominant-language inconsistency. MACRO STATE "Dominant Lang" named a documentation language (PLAINTEXT) while the COMPOSITION table showed JAVASCRIPT as the clear majority. Both read the same lang_comp dict but _get_dominant_lang ranks by summed structural `impact`, and documentation languages draw impact from a full-line-count basis (LOC/50), letting a few large plaintext files out-rank real code. _get_dominant_lang now takes the impact-argmax over code languages, excluding DOCUMENTATION_LANGUAGES, and only falls back to the full set for an all-docs repo -- keeping the deliberate impact-weighting while killing the false negative. 2. .js over-exclusion (same family as #2415/#2512). The aperture's Semantic Infrastructure & Test Target Shield dropped a real project's own lib/test/examples source because package.json only intent-locks its `main` entry. A manifest at the scan root now sets GuideStar.has_manifest_scope, propagated to ApertureFilter.manifest_project_scope, which stands down ONLY that shield (Gate 3) -- a separate signal from has_intent so the extension whitelist (Gate 1.5) and minification/machine-generated content gates stay active. node_modules/dist/vendor remain excluded via IGNORED_DIRECTORIES. Root-level detection keeps manifest-less scans (and the language-crucible corpus, scanned at a manifest-less data/ root) fully shielded. 3. Extension quoting artifact. `git ls-files` without -z wraps paths with unusual bytes in core.quotepath double-quotes, leaking a trailing `"` into the extracted extension and the exclusion reason ('.txt"'). Switch the census (galaxyscope) and the churn denominator (chronometer) to `git ls-files -z` and NUL-split; also defensively sanitize the aperture reason string so a malformed extension can never reach user-facing text. Tests: new tests/core_engine/test_lang_classification_2555.py (9 cases); existing git-output mocks updated to the -z contract. Full suite 8299 passed; golden crucible full_precision + zero_dependency both PASS (no master drift). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01SpojRfky4j31UMHLDcPdto --- gitgalaxy/core/aperture.py | 28 ++- gitgalaxy/core/guidestar_lens.py | 15 ++ gitgalaxy/galaxyscope.py | 14 +- gitgalaxy/metrics/chronometer.py | 7 +- gitgalaxy/metrics/signal_processor.py | 19 +- tests/core_engine/test_chronometer.py | 6 +- tests/core_engine/test_galaxyscope.py | 5 +- .../test_lang_classification_2555.py | 173 ++++++++++++++++++ 8 files changed, 254 insertions(+), 13 deletions(-) create mode 100644 tests/core_engine/test_lang_classification_2555.py diff --git a/gitgalaxy/core/aperture.py b/gitgalaxy/core/aperture.py index faea14f59..9fb9006af 100644 --- a/gitgalaxy/core/aperture.py +++ b/gitgalaxy/core/aperture.py @@ -125,6 +125,14 @@ def __init__( self._intent_cache: set[str] = set() self.dynamic_ignore_dirs: set[str] = set() + # #2555: whole-scan property, set once from GuideStar when a manifest sits at the + # scan root. It relaxes ONLY the Semantic Infrastructure & Test Target Shield + # (Gate 3 of _check_ignore_rules) -- deliberately NOT the extension whitelist + # (Gate 1.5) or the minification/machine-generated content gates, which a real + # project still wants applied. That is why this is a separate signal and not just + # a project-wide `has_intent=True` (intent short-circuits those other gates too). + self.manifest_project_scope: bool = False + self.logger.debug(f"Initializing Aperture Filter for project: '{self.root.name}'...") # Optimized Lookup Construction @@ -224,7 +232,13 @@ def evaluate_path_integrity(self, file_path: Union[str, Path], has_intent: bool if path_obj.name in self.exact_match_files or ext.lower() in self.whitelisted_extensions: return True, size_bytes, "Passed (Whitelisted)" - reason = f"Blocked (Unsupported Extension: '{ext}')" + # #2555: defensive -- never emit a malformed extension into user-facing text. + # The census is now dequoted upstream (git ls-files -z), but any path that + # reaches here carrying stray bytes (a quotepath artifact, a non-git census + # path) must not leak e.g. a trailing `"` into the reason. Mirror the REGEX + # SHIELD used by galaxyscope's anomaly summarizer. + safe_ext = ext if re.match(r"^\.[a-z0-9_\-+]+$", ext.lower()) else "no_extension" + reason = f"Blocked (Unsupported Extension: '{safe_ext}')" return False, size_bytes, reason def is_in_scope( @@ -538,8 +552,18 @@ def _check_ignore_rules(self, rel_path: str, has_intent: bool = False) -> bool: # 3. Semantic Infrastructure & Test Target Shield # SQL DDL/DML is exempt -- see _SQL_DDL_EXTENSIONS' own comment (#2512). + # #2555: a scan whose root carries a recognized manifest (manifest_project_scope) + # is a real project -- its own lib/test/examples dirs are first-class source, not + # generated/vendor noise, so the shield is stood down here (node_modules/dist/ + # vendor are still caught by the static IGNORED_DIRECTORIES gate above, and + # minified/machine-generated files by the content gates in is_in_scope). ext = Path(rel_path).suffix.lower() - if ext not in self._SQL_DDL_EXTENSIONS and self.infra_path_pattern.search(rel_path) and not has_intent: + if ( + ext not in self._SQL_DDL_EXTENSIONS + and self.infra_path_pattern.search(rel_path) + and not has_intent + and not self.manifest_project_scope + ): return False # 4. The Denylist (Vendor Blob Deflection) diff --git a/gitgalaxy/core/guidestar_lens.py b/gitgalaxy/core/guidestar_lens.py index e41b3b6b3..ddf5e258a 100644 --- a/gitgalaxy/core/guidestar_lens.py +++ b/gitgalaxy/core/guidestar_lens.py @@ -87,6 +87,16 @@ def __init__( # Spatial Documentation Map: Dict[directory_path, coverage_strength_float] self.documentation_coverage: dict[str, float] = {} + # #2555: True once a recognized package/build manifest is found AT THE SCAN ROOT. + # A root manifest is authoritative proof the scan target is a real project the + # user means to analyze, so its own first-class source/test/example directories + # (`lib/`, `test/`, `examples/`, ...) should not be silently dropped by the + # aperture's Semantic Infrastructure & Test Target Shield. Root-level only: + # a manifest nested in a subdirectory does NOT flip this, which is what keeps + # manifest-less directory scans (and the language-crucible corpus, scanned at a + # manifest-less `data/` root) fully shielded. + self.has_manifest_scope: bool = False + self.logger.debug(f"GuideStar Lens Online | Sector: {self.root.name}") def scan_project_config(self): @@ -206,6 +216,11 @@ def _scan_package_manifests(self): for manifest, lang in active_manifests.items(): path = self.root / manifest if path.exists(): + # #2555: a manifest at the scan root proves this is a real project -> + # grant project scope so its own source/test/example dirs survive the + # aperture's infra/test shield. Root-level only (path == self.root / ). + self.has_manifest_scope = True + # 1. Prioritize the manifest itself self._inject_intent_lock(manifest, lang, 0.90, "Roadmap Lock (Manifest)") diff --git a/gitgalaxy/galaxyscope.py b/gitgalaxy/galaxyscope.py index 2d7e38d76..7f7107e14 100644 --- a/gitgalaxy/galaxyscope.py +++ b/gitgalaxy/galaxyscope.py @@ -221,6 +221,8 @@ def _init_worker( ) _worker_state["guidestar"].scan_project_config() + # #2555: a root manifest stands down the aperture's infra/test shield for this scan. + _worker_state["filter"].manifest_project_scope = _worker_state["guidestar"].has_manifest_scope def _process_file_worker(rel_path: str) -> dict[str, Any]: @@ -971,6 +973,8 @@ def execute_pipeline(self, output_file: str = "galaxy.json"): # OS-level walk determining physical existence, OS permissions, and intent. t_phase = time.time() self.guidestar.scan_project_config() + # #2555: a root manifest stands down the aperture's infra/test shield for this scan. + self.filter.manifest_project_scope = self.guidestar.has_manifest_scope self._build_file_census() logger.debug(f"⏱️ EXECUTION_TIME [Phase 0 - Radar]: {time.time() - t_phase:.2f}s") @@ -1516,10 +1520,16 @@ def _build_file_census(self): git_paths = [self.single_file_target] self.git_tracked_files = set(git_paths) else: + # #2555: `-z` (NUL-delimited) output disables git's default + # core.quotepath octal-escaping/double-quote-wrapping of paths with + # "unusual" bytes (non-ASCII, tab, backslash, literal `"`). Without it, + # such a path arrives wrapped as `"src/na\303\257ve.txt"`, and the + # surrounding `"` leaks all the way into the extracted extension + # (`.txt"`) and the exclusion reason string. raw_output = subprocess.check_output( # noqa: S603 -- _GIT_BIN resolved absolute, args are fixed strings - [_GIT_BIN, "ls-files"], cwd=self.root, text=True, stderr=subprocess.DEVNULL + [_GIT_BIN, "ls-files", "-z"], cwd=self.root, text=True, stderr=subprocess.DEVNULL ) - git_paths = raw_output.splitlines() + git_paths = [p for p in raw_output.split("\0") if p] self.git_tracked_files = set(git_paths) # --- FAST I/O: ThreadPool for os.stat operations --- diff --git a/gitgalaxy/metrics/chronometer.py b/gitgalaxy/metrics/chronometer.py index a737c71ed..6ce9db25c 100644 --- a/gitgalaxy/metrics/chronometer.py +++ b/gitgalaxy/metrics/chronometer.py @@ -244,14 +244,17 @@ def _scan_git_history(self): # 1. Establish the Denominator (Total Tracked Files) try: + # #2555: `-z` disables core.quotepath quoting so paths with unusual bytes + # are not double-quote-wrapped (keeps this denominator count consistent with + # the census built in galaxyscope._build_file_census). res = subprocess.run( # noqa: S603 -- _GIT_BIN resolved absolute, fixed args - [_GIT_BIN, "ls-files"], + [_GIT_BIN, "ls-files", "-z"], cwd=self.root, capture_output=True, text=True, check=True, ) - tracked_files = set(res.stdout.splitlines()) + tracked_files = {p for p in res.stdout.split("\0") if p} total_files = len(tracked_files) except Exception as e: self.logger.warning(f"Chronometer: git ls-files failed ({e}). Aborting stream to prevent timeout trap.") diff --git a/gitgalaxy/metrics/signal_processor.py b/gitgalaxy/metrics/signal_processor.py index 60b59d1fe..36032e2f2 100644 --- a/gitgalaxy/metrics/signal_processor.py +++ b/gitgalaxy/metrics/signal_processor.py @@ -2112,5 +2112,20 @@ def _language_constants(self, lang_id: str) -> tuple[int, float, dict[str, float def _get_dominant_lang(self, composition: dict[str, dict[str, Any]]) -> str: if not composition: return "mixed" - # Sort by active structural impact instead of raw lines of code - return max(composition.items(), key=lambda x: x[1].get("impact", 0.0))[0] + # Sort by active structural impact instead of raw lines of code. + # + # #2555: documentation languages get their `file_impact` from a *full* + # line-count basis (`max(total_loc/50, 1.0)`, the STATIC LITERATURE OVERRIDE), + # so a handful of large plaintext/markdown files can out-rank real source on + # summed impact even while the COMPOSITION table (which ranks by file count / + # coding_loc) shows a code language as the clear majority -- producing a report + # that names PLAINTEXT dominant over JAVASCRIPT 53.8%. A documentation language + # is never the "dominant language" of a codebase that also contains code, so we + # take the impact-argmax over code languages first and only fall back to the + # full set (docs included) when there is no code language present at all. + doc_languages = { + lang.lower() for lang in self.asset_masks.get("DOCUMENTATION_LANGUAGES", {"markdown", "plaintext", "rst", "text"}) + } + code_langs = {lang: stats for lang, stats in composition.items() if lang.lower() not in doc_languages} + ranked = code_langs or composition + return max(ranked.items(), key=lambda x: x[1].get("impact", 0.0))[0] diff --git a/tests/core_engine/test_chronometer.py b/tests/core_engine/test_chronometer.py index 45c331845..70db83c9b 100644 --- a/tests/core_engine/test_chronometer.py +++ b/tests/core_engine/test_chronometer.py @@ -52,7 +52,7 @@ def git_side_effect(cmd, **kwargs): elif "log" in cmd and "abc123hash" in cmd: m.stdout = "1000\n" # Min Time elif "ls-files" in cmd: - m.stdout = "src/main.py\n" # Fake tracking + m.stdout = "src/main.py\0" # Fake tracking (#2555: git ls-files -z, NUL-delimited) return m mock_run.side_effect = git_side_effect @@ -91,8 +91,8 @@ def test_load_ignored_revs(tmp_path): @patch("gitgalaxy.metrics.chronometer.subprocess.Popen") def test_scan_git_history_and_stream(mock_popen, mock_run, tmp_path): """Proves the Popen stream handles quoted paths, skipped hashes, and empty lines.""" - # 1. Mock ls-files - mock_run.return_value = MagicMock(stdout="src/main.py\nsrc/utils.py\n") + # 1. Mock ls-files (#2555: git ls-files -z, NUL-delimited) + mock_run.return_value = MagicMock(stdout="src/main.py\0src/utils.py\0") # 2. Mock Popen stream output with hostile edge cases mock_process = MagicMock() diff --git a/tests/core_engine/test_galaxyscope.py b/tests/core_engine/test_galaxyscope.py index 850c12da7..171957a20 100644 --- a/tests/core_engine/test_galaxyscope.py +++ b/tests/core_engine/test_galaxyscope.py @@ -595,9 +595,10 @@ def test_micro_mass_quota_exhaustion(self, mock_aperture, mock_git): scope = Orchestrator(".", self.mock_config) scope.MICRO_MASS_GRACE_LIMIT = 5 # Lower for faster testing - # Mock Git returning 10 tiny files in the same directory + # Mock Git returning 10 tiny files in the same directory. + # #2555: census now consumes `git ls-files -z` (NUL-delimited) output. fake_files = [f"src/assets/icon_{i}.svg" for i in range(10)] - mock_git.return_value = "\n".join(fake_files) + mock_git.return_value = "\0".join(fake_files) + "\0" # Mock aperture returning: is_valid=True, size=10 bytes (under MICRO_MASS_BYTES) mock_aperture.return_value = (True, 10, "Passed") diff --git a/tests/core_engine/test_lang_classification_2555.py b/tests/core_engine/test_lang_classification_2555.py new file mode 100644 index 000000000..098365425 --- /dev/null +++ b/tests/core_engine/test_lang_classification_2555.py @@ -0,0 +1,173 @@ +""" +#2555 -- Language-classification consistency regression tests. + +Covers the three defects reported against a single `galaxyscope --llm-only` scan of +expressjs/express: + + 1. Dominant-language inconsistency: the MACRO STATE "Dominant Lang" named a + documentation language (PLAINTEXT) while the COMPOSITION table showed a code + language (JAVASCRIPT 53.8%) as the clear majority. + 2. `.js` over-exclusion: the aperture's Semantic Infrastructure & Test Target Shield + dropped a real project's own lib/test/examples source because a recognized manifest + did not confer project scope. + 3. Extension quoting artifact: a `git ls-files` quotepath artifact leaked a stray + trailing `"` into the extracted extension and the exclusion reason string. +""" + +import json + +import pytest + +from gitgalaxy.core.aperture import ApertureFilter +from gitgalaxy.core.guidestar_lens import GuideStarLens +from gitgalaxy.metrics.signal_processor import SignalProcessor + +# ============================================================================== +# MOCK CALIBRATION +# ============================================================================== +MOCK_REGISTRY = { + "javascript": {"extensions": [".js"], "exact_matches": []}, + "python": {"extensions": [".py"], "exact_matches": []}, + "markdown": {"extensions": [".md"], "exact_matches": ["README.md"]}, +} + +MOCK_APERTURE_CONFIG = { + "SECRETS_EXACT": set(), + "SECRETS_EXTENSIONS": set(), + "MAX_FILE_SIZE_MB": 10, + "MAX_FILE_SIZE_HARD_MB": 100, + "MAX_LINE_LENGTH": 500, + "IGNORED_DIRECTORIES": {"node_modules", ".git"}, + "IGNORED_EXTENSIONS": {".exe", ".dll"}, + "CONTRABAND_PATTERNS": ["*-min.js", "*.bundle.js"], +} + +MOCK_GUIDESTAR_CONFIG = { + "MANIFEST_MAP": { + "package.json": "javascript", + "pyproject.toml": "python", + }, + "INTENT_BIASED_SECTORS": ["src"], + "EXEC_PREFIX_MAP": {}, +} + + +@pytest.fixture +def filter_engine(tmp_path): + return ApertureFilter( + root_dir=tmp_path, + language_definitions=MOCK_REGISTRY, + aperture_config=MOCK_APERTURE_CONFIG, + ) + + +# ============================================================================== +# DEFECT 1: DOMINANT LANG MUST NOT BE A DOCUMENTATION LANGUAGE OVER REAL CODE +# ============================================================================== +def test_dominant_lang_ignores_documentation_when_code_present(): + """ + A handful of large plaintext files can out-rank code on summed structural `impact` + (docs get impact from a full-line-count basis). The dominant language of a codebase + that contains code must be a code language, matching the COMPOSITION table. + """ + proc = SignalProcessor() + composition = { + # plaintext wins on raw impact, but is documentation + "plaintext": {"files": 5, "loc": 1, "impact": 500.0}, + # javascript is the real majority (highest files/loc), lower impact + "javascript": {"files": 43, "loc": 1167, "impact": 120.0}, + "python": {"files": 2, "loc": 40, "impact": 30.0}, + } + assert proc._get_dominant_lang(composition) == "javascript" + + +def test_dominant_lang_picks_highest_impact_code_lang(): + """Among code languages the deliberate impact-weighting still decides the winner.""" + proc = SignalProcessor() + composition = { + "markdown": {"files": 10, "loc": 9000, "impact": 999.0}, # docs -> excluded + "python": {"files": 3, "loc": 50, "impact": 5.0}, + "c": {"files": 2, "loc": 30, "impact": 50.0}, # highest-impact code lang + } + assert proc._get_dominant_lang(composition) == "c" + + +def test_dominant_lang_falls_back_to_docs_when_no_code(): + """An all-documentation repo may legitimately report a documentation language.""" + proc = SignalProcessor() + composition = { + "plaintext": {"files": 4, "loc": 10, "impact": 40.0}, + "markdown": {"files": 6, "loc": 20, "impact": 90.0}, + } + assert proc._get_dominant_lang(composition) == "markdown" + + +def test_dominant_lang_empty_is_mixed(): + assert SignalProcessor()._get_dominant_lang({}) == "mixed" + + +# ============================================================================== +# DEFECT 2: MANIFEST-AWARE PROJECT SCOPE +# ============================================================================== +def test_root_manifest_grants_project_scope(tmp_path): + """A manifest at the scan root flips has_manifest_scope.""" + (tmp_path / "package.json").write_text(json.dumps({"main": "index.js"}), encoding="utf-8") + lens = GuideStarLens(root_path=tmp_path, guidestar_config=MOCK_GUIDESTAR_CONFIG) + lens.scan_project_config() + assert lens.has_manifest_scope is True + + +def test_nested_manifest_does_not_grant_project_scope(tmp_path): + """A manifest buried in a subdirectory must NOT confer whole-scan project scope -- + this is what keeps manifest-less corpus scans (e.g. language-crucible's data/ root) + fully shielded.""" + nested = tmp_path / "packages" / "widget" + nested.mkdir(parents=True) + (nested / "package.json").write_text(json.dumps({"main": "index.js"}), encoding="utf-8") + lens = GuideStarLens(root_path=tmp_path, guidestar_config=MOCK_GUIDESTAR_CONFIG) + lens.scan_project_config() + assert lens.has_manifest_scope is False + + +def test_project_scope_stands_down_infra_shield_but_keeps_ignored_dirs(filter_engine): + """With project scope, a real project's own lib/test/examples source survives the + infra/test shield, but node_modules/vendor are still dropped by IGNORED_DIRECTORIES.""" + shielded_source = ("lib/index.js", "test/app.test.js", "examples/mvc/lib/boot.js", "spec/foo_spec.js") + + # Default (no manifest scope): the shield drops first-class source. + for path in shielded_source: + assert filter_engine._check_ignore_rules(path) is False, path + + # Project scope stands the shield down. + filter_engine.manifest_project_scope = True + for path in shielded_source: + assert filter_engine._check_ignore_rules(path) is True, path + + # ...but hard-ignored directories are still excluded (independent gate). + for still_blocked in ("node_modules/express/index.js", "src/app.bundle.js"): + assert filter_engine._check_ignore_rules(still_blocked) is False, still_blocked + + +# ============================================================================== +# DEFECT 3: EXTENSION QUOTING ARTIFACT MUST NOT LEAK INTO THE REASON STRING +# ============================================================================== +def test_malformed_extension_not_leaked_into_reason(filter_engine, tmp_path): + """A path carrying a stray quote (quotepath artifact) must not surface a malformed + extension like `.txt"` in the human-readable exclusion reason.""" + bad = tmp_path / 'naive.txt"' + bad.write_text("data", encoding="utf-8") + + is_valid, _, reason = filter_engine.evaluate_path_integrity(bad, has_intent=False) + assert is_valid is False + assert '"' not in reason, reason + assert "no_extension" in reason, reason + + +def test_clean_extension_still_reported(filter_engine, tmp_path): + """A genuinely unsupported but well-formed extension is reported verbatim.""" + f = tmp_path / "notes.xyz" + f.write_text("data", encoding="utf-8") + + is_valid, _, reason = filter_engine.evaluate_path_integrity(f, has_intent=False) + assert is_valid is False + assert ".xyz" in reason