From 22b4946003f7e2715dac39a8c33cdf04b52df34b Mon Sep 17 00:00:00 2001 From: Renn F Date: Wed, 14 Oct 2026 22:36:54 +0200 Subject: [PATCH 1/5] feat(detection): yield the event loop during scans, cap per-request scan time, add semantic-only header opt-out Await the shared regex executor in the async engine and yield between pattern slices so one request's scan no longer freezes the event loop (sync mirror keeps blocking). Add detection_max_scan_seconds (default 10.0), a per-request total regex scan budget carried in request-scoped state. When spent, remaining patterns and values are skipped and a weighted pattern_timeout threat (scan_time_budget_exhausted) is recorded like a per-pattern timeout. Add excluded_semantic_detection_headers (and detection_exclusion(semantic_headers=...)): opt-in header names whose values skip the semantic analyzer only; regex scanning is unchanged. --- docs/adapters/decorators.md | 1 + docs/api/decorators.md | 4 +- docs/api/models.md | 3 + docs/configuration/detection-tuning.md | 14 ++ docs/configuration/security-config.md | 2 + docs/internals/detection-engine.md | 2 + .../skills/guard-core/references/config.md | 4 +- .../skills/guard-core/references/detection.md | 2 +- .../_security_config_field_validators.py | 5 + guard_core/_security_config_fields.py | 23 +++ guard_core/_utils/detection_config.py | 29 ++++ guard_core/_utils/detection_scan.py | 88 ++++++++++ guard_core/_utils/penetration_detection.py | 11 +- guard_core/decorators/content_filtering.py | 3 + guard_core/decorators/route_config.py | 1 + guard_core/detection_engine/compiler.py | 14 ++ guard_core/handlers/_suspatterns_regex.py | 37 ++++- guard_core/handlers/suspatterns_handler.py | 9 +- guard_core/sync/_utils/detection_config.py | 29 ++++ guard_core/sync/_utils/detection_scan.py | 88 ++++++++++ .../sync/_utils/penetration_detection.py | 11 +- .../sync/decorators/content_filtering.py | 3 + guard_core/sync/decorators/route_config.py | 1 + guard_core/sync/detection_engine/compiler.py | 9 + .../sync/handlers/_suspatterns_regex.py | 37 ++++- .../sync/handlers/suspatterns_handler.py | 9 +- scripts/unasync.py | 30 ++++ .../test_detection_exclusion.py | 13 ++ ...t_every_collection_field_is_revalidated.py | 1 + tests/test_scan_event_loop_yield.py | 110 +++++++++++++ tests/test_scan_time_budget.py | 153 +++++++++++++++++ tests/test_semantic_excluded_headers.py | 103 ++++++++++++ tests/test_sensitive_data_invariant.py | 18 ++ tests/test_sensitive_output_producers.py | 3 + .../test_pattern_source_redaction.py | 16 +- ...ll_dash_c_var_assignment_newline_prefix.py | 10 +- tests/test_sus_patterns/test_sus_patterns.py | 33 ++-- .../test_detection_exclusion.py | 13 ++ ...t_every_collection_field_is_revalidated.py | 1 + tests/test_sync/test_scan_time_budget.py | 155 ++++++++++++++++++ .../test_semantic_excluded_headers.py | 105 ++++++++++++ .../test_sensitive_data_invariant.py | 18 ++ .../test_sensitive_output_producers.py | 3 + .../test_pattern_source_redaction.py | 16 +- ...ll_dash_c_var_assignment_newline_prefix.py | 10 +- 45 files changed, 1187 insertions(+), 63 deletions(-) create mode 100644 tests/test_scan_event_loop_yield.py create mode 100644 tests/test_scan_time_budget.py create mode 100644 tests/test_semantic_excluded_headers.py create mode 100644 tests/test_sync/test_scan_time_budget.py create mode 100644 tests/test_sync/test_semantic_excluded_headers.py diff --git a/docs/adapters/decorators.md b/docs/adapters/decorators.md index 30f1a0a9..4893d04b 100644 --- a/docs/adapters/decorators.md +++ b/docs/adapters/decorators.md @@ -50,6 +50,7 @@ class RouteConfig: self.api_key_required: bool = False self.geo_rate_limits: dict[str, tuple[int, int]] | None = None self.excluded_detection_headers: set[str] | None = None + self.excluded_semantic_detection_headers: set[str] | None = None self.excluded_detection_params: set[str] | None = None self.excluded_detection_body_fields: set[str] | None = None self.enabled_detection_categories: set[str] | None = None diff --git a/docs/api/decorators.md b/docs/api/decorators.md index 64b7291a..598d9a9c 100644 --- a/docs/api/decorators.md +++ b/docs/api/decorators.md @@ -156,15 +156,17 @@ def detection_exclusion( body_fields: set[str] | None = None, categories: set[str] | None = None, scan_body: bool | None = None, + semantic_headers: set[str] | None = None, ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: ... ``` -All five kwargs are optional (`set[str] | None` for `headers`/`params`/`body_fields`/`categories`, `bool | None` for `scan_body`). Passing `None` (or omitting) leaves the corresponding `RouteConfig` field unset, the route inherits the global `SecurityConfig` value at request time. Passing a value replaces the inherited value at this route only. +All six kwargs are optional (`set[str] | None` for `headers`/`params`/`body_fields`/`categories`/`semantic_headers`, `bool | None` for `scan_body`). Passing `None` (or omitting) leaves the corresponding `RouteConfig` field unset, the route inherits the global `SecurityConfig` value at request time. Passing a value replaces the inherited value at this route only. - `headers`, header names skipped by detection. Merged with `SecurityConfig.excluded_detection_headers` and the hardcoded default exclusion list. - `params`, query parameter names skipped by detection. Replaces (does not merge with) the global set when set. - `body_fields`, top-level JSON body keys skipped by detection. Replaces the global set when set. - `categories`, categories the regex scanner runs at this route. Replaces the global `enabled_detection_categories`. Custom user patterns always run regardless. +- `semantic_headers`, header names whose values skip the semantic analyzer only (regex scanning is unchanged). Merged with `SecurityConfig.excluded_semantic_detection_headers`. - `scan_body`, whether to scan the request body at this route. Replaces the global `detection_scan_body` when set. ```python diff --git a/docs/api/models.md b/docs/api/models.md index 1cd29eb9..f343562b 100644 --- a/docs/api/models.md +++ b/docs/api/models.md @@ -48,6 +48,7 @@ class SecurityConfig(BaseModel): global_behavior_rules: tuple[BehaviorRuleConfig, ...] = Field(default_factory=tuple) excluded_detection_headers: set[str] = Field(default_factory=set) + excluded_semantic_detection_headers: set[str] = Field(default_factory=set) excluded_detection_params: set[str] = Field(default_factory=set) excluded_detection_body_fields: set[str] = Field(default_factory=set) enabled_detection_categories: frozenset[str] = Field( @@ -135,6 +136,7 @@ class SecurityConfig(BaseModel): endpoint_rate_limits: dict[str, tuple[int, int]] = Field(default_factory=dict) detection_compiler_timeout: float = Field(default=2.0, ge=0.1, le=10.0) + detection_max_scan_seconds: float = Field(default=10.0, ge=0.1, le=60.0) detection_max_content_length: int = Field(default=10000, ge=1000, le=100000) detection_max_body_inspect_bytes: int = Field(default=262144, ge=1024, le=10485760) detection_preserve_attack_patterns: bool = Field(default=True) @@ -184,6 +186,7 @@ These fields opt specific request components out of penetration detection. The h | Field | Type | Default | Description | |------------------------------------|-------------|----------------------------------|----------------------------------------------------------------------------| | `excluded_detection_headers` | `set[str]` | `set()` | Headers exempted only from the categories known to false-positive on their typical values: identity and proxy headers such as `X-Forwarded-For`, `X-Real-IP`, `Host`, `Origin` and `Via` skip `ssrf` only, and every other category (sqli, xss, cmd_injection, template, Log4Shell) still scans them. A name outside the built-in identity set inherits the same rule when its value looks like an address chain, otherwise it gets no exclusion at all. Merged with the hardcoded default list. | +| `excluded_semantic_detection_headers` | `set[str]` | `set()` | Opt-in. Header names (case-insensitive) whose values skip the semantic analyzer only; regex category scanning of these headers is unchanged, and `excluded_detection_headers` semantics are untouched. Use it when a header legitimately carries high-entropy encoded data, for example `cookie` or `authorization` for apps with JWT sessions, whose base64 segments score close to `detection_semantic_threshold`. Merged with the route-level set from `@security.detection_exclusion(semantic_headers=...)`. | | `excluded_detection_params` | `set[str]` | `set()` | Query parameter names skipped by detection. | | `excluded_detection_body_fields` | `set[str]` | `set()` | Top-level JSON body keys skipped by detection. | | `enabled_detection_categories` | `frozenset[str]` | `frozenset(ALL_DETECTION_CATEGORIES)` | Categories scanned for. Validator rejects unknown labels. | diff --git a/docs/configuration/detection-tuning.md b/docs/configuration/detection-tuning.md index b3833417..ac246b15 100644 --- a/docs/configuration/detection-tuning.md +++ b/docs/configuration/detection-tuning.md @@ -25,6 +25,14 @@ Maximum time in seconds for a single regex pattern match. Patterns that exceed t | `2.0` | Balanced. Catches most attacks while limiting resource usage.| | `5.0` | Permissive. Better detection but higher latency risk. | +### `detection_max_scan_seconds` + +**Type**: `float` | **Default**: `10.0` | **Range**: 0.1 - 60.0 + +Total regex scan time allowed for one request, summed across every field (query parameters, headers, body values) and every pattern, not per `detect()` call. Once the budget is spent, the remaining patterns and request values are not scanned and the request gets a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the reported `timeouts`, the same way a per-pattern `detection_compiler_timeout` does, so padding a request with filler cannot silently skip detection. The wait for a single pattern is also clamped to the time left in the budget. The default is far above normal traffic (a few milliseconds per request, roughly 16 ms for 8,863 characters on one measured CPU) and about 6 times the worst legitimate case the value cap allows (256 benign query parameters, 512 scan units, measured at about 1.5 s on that CPU), which leaves headroom for slower hosts. Only regex pattern time counts, not decoding or semantic analysis. + +In the async tree the scan awaits the regex worker instead of blocking the event loop, so other requests run between patterns. A single regex that is already executing still holds the GIL until it returns. The sync tree has no event loop and keeps blocking. + ### `detection_pattern_validation_cache_path` **Type**: `str | None` | **Default**: `None` @@ -82,6 +90,12 @@ Minimum score from the `SemanticAnalyzer` to classify content as a threat. The s !!! info "Semantic vs Regex" Regex patterns provide definitive threat detection. Semantic analysis is a secondary layer that catches obfuscated or novel attacks. Lowering the semantic threshold increases the chance of catching evasion attempts but also increases false positives on legitimate content containing technical terms. +### `excluded_semantic_detection_headers` + +**Type**: `set[str]` | **Default**: `set()` + +Opt-in list of header names whose values skip the semantic analyzer only, matched case-insensitively. Regex category scanning of those headers is unchanged, and `excluded_detection_headers` keeps its meaning (it never silences a header). The semantic analyzer counts base64 as obfuscation and encoding layers, so JWT session cookies and bearer tokens can score close to `detection_semantic_threshold` (0.30 to 0.59 measured on sample cookies, 0.71 reported). For apps with JWT sessions, set `excluded_semantic_detection_headers={"cookie", "authorization"}` instead of raising the global threshold. A route can add names with `@security.detection_exclusion(semantic_headers={...})`. + ### `detection_anomaly_threshold` **Type**: `float` | **Default**: `3.0` | **Range**: 1.0 - 10.0 diff --git a/docs/configuration/security-config.md b/docs/configuration/security-config.md index d17aa408..ab9157c2 100644 --- a/docs/configuration/security-config.md +++ b/docs/configuration/security-config.md @@ -180,6 +180,7 @@ These fields opt request components out of penetration detection. The header set | Field | Type | Default | Description | |------------------------------------|-------------|----------------------------------|----------------------------------------------------------------------------| | `excluded_detection_headers` | `set[str]` | `set()` | Headers exempted only from the categories known to false-positive on their typical values: identity and proxy headers such as `X-Forwarded-For`, `X-Real-IP`, `Host`, `Origin` and `Via` skip `ssrf` only, and every other category (sqli, xss, cmd_injection, template, Log4Shell) still scans them. A name outside the built-in identity set inherits the same rule when its value looks like an address chain, otherwise it gets no exclusion at all. Merged with the hardcoded default list. | +| `excluded_semantic_detection_headers` | `set[str]` | `set()` | Opt-in. Header names (case-insensitive) whose values skip the semantic analyzer only; regex category scanning of these headers is unchanged, and `excluded_detection_headers` semantics are untouched. Use it when a header legitimately carries high-entropy encoded data, for example `cookie` or `authorization` for apps with JWT sessions, whose base64 segments score close to `detection_semantic_threshold`. Merged with the route-level set from `@security.detection_exclusion(semantic_headers=...)`. | | `excluded_detection_params` | `set[str]` | `set()` | Query parameter names skipped by detection. | | `excluded_detection_body_fields` | `set[str]` | `set()` | Top-level JSON body keys skipped by detection. | | `enabled_detection_categories` | `frozenset[str]` | full `ALL_DETECTION_CATEGORIES` | Categories scanned for. Validator rejects unknown labels. | @@ -393,6 +394,7 @@ Detection Engine | `detection_max_tracked_patterns` | `int` | `1000` | 100 - 5000 | Maximum patterns to track for performance. | | `detection_max_body_inspect_bytes` | `int` | `262144`| 1024 - 10485760 | Body size cap read/scanned for detection; distinct from `detection_max_content_length` and `max_request_size`. A body whose declared `Content-Length` exceeds this cap is read through the adapter's bounded reader (`read_body_prefix`) and only the first this-many bytes are scanned, if the adapter implements one; without a bounded reader the body is not read at all, the same skip as before this cap existed. Either way a one-time warning names the cap, the client, and which of the two happened. A signature that straddles the cap boundary is still caught: a small overlap (the longest compiled pattern length, up to 256 bytes) is read past the cap on every path, including a declared oversized `Content-Length`. | | `detection_max_scan_values` | `int` | `512` | 2 - 100000 | Maximum values (query params, headers, JSON keys/values, form/multipart fields) scanned per request; remaining values are skipped and a one-time warning logs the client IP once reached. Each named value costs two scan units (name, then value), so the minimum is 2. | +| `detection_max_scan_seconds` | `float` | `10.0` | 0.1 - 60.0 | Cap on the total regex scan time spent on one request, summed across every field and pattern (not per `detect()` call). Once reached, remaining patterns and values are not scanned and the request receives a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the event `timeouts`, so padding a request cannot skip detection; a one-time warning logs the client IP. A single pattern's wait is also clamped to the time left. Only regex pattern time is counted. | | `detection_max_scan_chars` | `int` | `65536` | 1024 - 262144 | Maximum total characters, across every value handed to the pattern engine per request (counted at the same point as `detection_max_scan_values`), before remaining values are skipped and a one-time warning logs the client IP. A value already in progress when the budget is checked is always scanned in full; only values that would start after the budget is spent are skipped, so a single large value is never silently dropped (GHSA-3hfx-8m47-5f9h residual). | | `detection_max_json_depth` | `int` | `32` | 1 - 1000 | Maximum nesting depth of a JSON request body walked structurally. A dict or list reached at this depth is not descended into: it is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, so content hidden below the cap is still scanned as text rather than structurally. A one-time warning logs the client IP once reached (GHSA-f6cf-jjhc-qp85). | | `detection_binary_min_run_length` | `int` | `16` | 4 - 1024 | Minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned. A part payload whose binary artifact characters make up at least a fifth of it is reduced to its printable runs before scanning; compressed bytes produce attack-shaped printable fragments at a rate that grows with file size, so shorter runs are skipped instead of matched. Text uploads and short or mostly-text payloads stay below the ratio and keep their full scan, whole-body fallback scans are never reduced (raw-body signature coverage stays intact), and text embedded in uploads (scripts inside PDFs, stored paths in archives) forms runs past the threshold and is scanned in full. File names, multipart field names, and part headers are scanned regardless (see `docs/configuration/detection-tuning.md`). | diff --git a/docs/internals/detection-engine.md b/docs/internals/detection-engine.md index 963a2cf8..9cbeae14 100644 --- a/docs/internals/detection-engine.md +++ b/docs/internals/detection-engine.md @@ -48,6 +48,8 @@ flowchart TD Built-in (compile-time-vetted) patterns match directly via `pattern.search()`, with no per-match thread or timeout. Only custom patterns (added via `add_pattern(..., custom=True)`) are routed through the safe-matcher timeout wrapper, a custom-pattern match is flagged as timed out once elapsed time reaches `0.9 * compiler.default_timeout` (i.e. `0.9 * detection_compiler_timeout` when a `SecurityConfig` is wired; legacy/no-compiler mode falls back to a hardcoded 2.0s field default). +The async tree awaits each shared-executor future (`_await_regex_future` in `guard_core/detection_engine/compiler.py`) rather than calling `future.result`, so the event loop runs other tasks between patterns; the sync mirror keeps the blocking `future.result` (the unasync script swaps the helper). A per-request regex time budget (`detection_max_scan_seconds`, tracked in request-scoped context variables by `_scan_value_budget`) stops the pattern loop once spent and emits a weighted `pattern_timeout` threat named `scan_time_budget_exhausted`; `excluded_semantic_detection_headers` skips only the semantic analyzer for the named headers. + When no `SecurityConfig` is wired at construction (legacy mode), `SusPatternsManager` builds neither a `PatternCompiler` nor a `ContentPreprocessor`; content is instead truncated to `detection_max_content_length` (default 10000) via direct slicing before matching, and every pattern, including what would otherwise be a "custom" pattern, runs through the same shared-executor timeout path described above (`_check_pattern_with_timeout`), reporting to the same success/timeout counters. Running detection unconfigured this way is deprecated and will be removed in a future major release: the first time `detect()` actually executes in this state, it raises a `DeprecationWarning` and logs an identical `logger.warning` naming the reason (`_LEGACY_DETECTION_WARNING`, checked by the `_warn_if_legacy_detection` helper in `guard_core/handlers/suspatterns_handler.py`), once per process, a module-level flag shared by every `SusPatternsManager` instance, not a per-instance one. The check runs from inside `detect()` itself, not at construction or at `configure()`, since the module-level singleton is always built unconfigured at import time and only gets a `SecurityConfig` later from the pipeline; a normally initialised pipeline configures the singleton (directly via `sus_patterns_handler.configure(config)`, or implicitly through `detect_penetration_attempt(request, config)`) before it ever runs detection, so it never warns. ___ diff --git a/guard_core/.agents/skills/guard-core/references/config.md b/guard_core/.agents/skills/guard-core/references/config.md index 404cd4bf..b8f282e3 100644 --- a/guard_core/.agents/skills/guard-core/references/config.md +++ b/guard_core/.agents/skills/guard-core/references/config.md @@ -39,11 +39,11 @@ In-memory rate-limit stores (`RateLimitManager.request_timestamps` and the modul ### Detection -`detection_compiler_timeout` (default `2.0`, range `0.1`-`10.0`; per-pattern match timeout), `detection_pattern_validation_cache_path` (default `None`; opt-in file path for a disk-backed pattern-validation cache: the empirical cost-verdict outcome of `validate_pattern_safety` is cached keyed by pattern, flags, and engine version so boots reuse prior certifications instead of re-timing every custom pattern, while the deterministic layers always re-run; entries from a different engine version are ignored and overwritten -- `guard_core/detection_engine/_validation_cache.py`), `detection_max_content_length` (default `10000`), `detection_max_body_inspect_bytes` (default `262144`; a body whose declared `Content-Length` exceeds this is read through the adapter's bounded reader, `read_body_prefix`, and only the first this-many bytes are scanned, if the adapter implements one -- without a bounded reader the body is not read at all, the same skip as before this cap existed -- with a one-time `logger.warning` naming the cap, the client, and which of the two happened), `detection_max_scan_values` (default `512`, `ge=2`, `le=100_000`; caps the total query-parameter, header, JSON-key/value, form-field, and multipart-part values scanned per request, including JSON embedded in a single value; each named value costs two scan units, name and value, so `2` is the floor; once reached, remaining request values are not scanned and a one-time `logger.warning` names the client IP -- `guard_core/_security_config_fields.py`, `guard_core/_utils/detection_scan.py`), `detection_max_scan_chars` (default `65536`, `ge=1024`, `le=262144`; caps the total characters across every value handed to the pattern engine per request, at the same accounting point as `detection_max_scan_values`; a value already in progress when checked is always scanned in full, only values that would start after the budget is spent are skipped, so a single large value is never silently dropped -- GHSA-3hfx-8m47-5f9h residual), `detection_max_json_depth` (default `32`, `ge=1`, `le=1000`; caps how deep the structural JSON-body walk descends, an explicit-stack iteration with no recursion; a dict or list reached at this depth is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, and a one-time `logger.warning` names the client IP -- GHSA-f6cf-jjhc-qp85, CWE-674, `guard_core/_security_config_fields.py`, `guard_core/_utils/body_content_scan.py`), `detection_binary_min_run_length` (default `16`, `ge=4`, `le=1024`; minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned: a part payload whose binary artifact characters fill at least a fifth of it is reduced to its printable runs, while text uploads, short or mostly-text payloads, text parts without a filename, and whole-body fallback scans keep their full scan so raw-body signature coverage (pickle opcodes, UTF-16/32 wide-encoding payloads, null-byte LDAP shapes) stays intact; text embedded in a binary-dense upload forms runs past the threshold and is scanned in full, while attack-shaped fragments shorter than the run inside compressed data are skipped as statistical noise -- `guard_core/detection_engine/binary_islands.py`, `guard_core/_utils/body_form_scan.py`, `guard_core/_utils/detection_scan.py`), `detection_preserve_attack_patterns` (default `True`), `detection_semantic_threshold` (default `0.7`), `detection_anomaly_threshold` (default `3.0`), `detection_slow_pattern_threshold` (default `0.1`), `detection_monitor_history_size` (default `1000`), `detection_max_tracked_patterns` (default `1000`), `detection_threat_score_threshold` (default `1.0`), `detection_scan_body` (default `True`), `enabled_detection_categories` (`frozenset[str]`, subset of the 19 categories; defaults to all 19), `enable_penetration_detection` (default `True`), `enable_dynamic_rules` (default `False`), `dynamic_rules_cache_path` (default `None`; `Path | None`, opt-in file path for the last-known dynamic rules snapshot that is persisted on every successful apply and hydrated once at `initialize_agent`, Redis first then the file; see the SecurityConfig section of SKILL.md; the dump keeps only the fields guard-core's own `DynamicRules` mirror models, so guard-agent-only fields on the live rules object, `emergency_whitelist_only`, `message`, never break persistence), `body_read_timeout` (default `3.0`, `gt=0.0`, `le=30.0`; seconds an adapter's `read_body_prefix`/`body` call is given before the body is treated as unavailable -- bounds the async tree via `asyncio.wait_for`, and in the sync tree bounds how long the caller joins the daemon thread the read runs on), `sync_body_read_max_concurrent` (default `64`, `ge=1`, `le=10000`; cap on daemon threads the sync tree may have blocked at once inside an adapter body read). See `docs/configuration/detection-tuning.md` in the guard-core repository for the full field-by-field tuning guide. +`detection_compiler_timeout` (default `2.0`, range `0.1`-`10.0`; per-pattern match timeout), `detection_max_scan_seconds` (default `10.0`, range `0.1`-`60.0`; total regex scan time per request across all fields and patterns; once spent, remaining patterns and values are not scanned and a weighted `pattern_timeout` threat named `scan_time_budget_exhausted` is added, counted in the anomaly score and `timeouts` -- `guard_core/_utils/detection_scan.py`), `detection_pattern_validation_cache_path` (default `None`; opt-in file path for a disk-backed pattern-validation cache: the empirical cost-verdict outcome of `validate_pattern_safety` is cached keyed by pattern, flags, and engine version so boots reuse prior certifications instead of re-timing every custom pattern, while the deterministic layers always re-run; entries from a different engine version are ignored and overwritten -- `guard_core/detection_engine/_validation_cache.py`), `detection_max_content_length` (default `10000`), `detection_max_body_inspect_bytes` (default `262144`; a body whose declared `Content-Length` exceeds this is read through the adapter's bounded reader, `read_body_prefix`, and only the first this-many bytes are scanned, if the adapter implements one -- without a bounded reader the body is not read at all, the same skip as before this cap existed -- with a one-time `logger.warning` naming the cap, the client, and which of the two happened), `detection_max_scan_values` (default `512`, `ge=2`, `le=100_000`; caps the total query-parameter, header, JSON-key/value, form-field, and multipart-part values scanned per request, including JSON embedded in a single value; each named value costs two scan units, name and value, so `2` is the floor; once reached, remaining request values are not scanned and a one-time `logger.warning` names the client IP -- `guard_core/_security_config_fields.py`, `guard_core/_utils/detection_scan.py`), `detection_max_scan_chars` (default `65536`, `ge=1024`, `le=262144`; caps the total characters across every value handed to the pattern engine per request, at the same accounting point as `detection_max_scan_values`; a value already in progress when checked is always scanned in full, only values that would start after the budget is spent are skipped, so a single large value is never silently dropped -- GHSA-3hfx-8m47-5f9h residual), `detection_max_json_depth` (default `32`, `ge=1`, `le=1000`; caps how deep the structural JSON-body walk descends, an explicit-stack iteration with no recursion; a dict or list reached at this depth is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, and a one-time `logger.warning` names the client IP -- GHSA-f6cf-jjhc-qp85, CWE-674, `guard_core/_security_config_fields.py`, `guard_core/_utils/body_content_scan.py`), `detection_binary_min_run_length` (default `16`, `ge=4`, `le=1024`; minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned: a part payload whose binary artifact characters fill at least a fifth of it is reduced to its printable runs, while text uploads, short or mostly-text payloads, text parts without a filename, and whole-body fallback scans keep their full scan so raw-body signature coverage (pickle opcodes, UTF-16/32 wide-encoding payloads, null-byte LDAP shapes) stays intact; text embedded in a binary-dense upload forms runs past the threshold and is scanned in full, while attack-shaped fragments shorter than the run inside compressed data are skipped as statistical noise -- `guard_core/detection_engine/binary_islands.py`, `guard_core/_utils/body_form_scan.py`, `guard_core/_utils/detection_scan.py`), `detection_preserve_attack_patterns` (default `True`), `detection_semantic_threshold` (default `0.7`), `detection_anomaly_threshold` (default `3.0`), `detection_slow_pattern_threshold` (default `0.1`), `detection_monitor_history_size` (default `1000`), `detection_max_tracked_patterns` (default `1000`), `detection_threat_score_threshold` (default `1.0`), `detection_scan_body` (default `True`), `enabled_detection_categories` (`frozenset[str]`, subset of the 19 categories; defaults to all 19), `enable_penetration_detection` (default `True`), `enable_dynamic_rules` (default `False`), `dynamic_rules_cache_path` (default `None`; `Path | None`, opt-in file path for the last-known dynamic rules snapshot that is persisted on every successful apply and hydrated once at `initialize_agent`, Redis first then the file; see the SecurityConfig section of SKILL.md; the dump keeps only the fields guard-core's own `DynamicRules` mirror models, so guard-agent-only fields on the live rules object, `emergency_whitelist_only`, `message`, never break persistence), `body_read_timeout` (default `3.0`, `gt=0.0`, `le=30.0`; seconds an adapter's `read_body_prefix`/`body` call is given before the body is treated as unavailable -- bounds the async tree via `asyncio.wait_for`, and in the sync tree bounds how long the caller joins the daemon thread the read runs on), `sync_body_read_max_concurrent` (default `64`, `ge=1`, `le=10000`; cap on daemon threads the sync tree may have blocked at once inside an adapter body read). See `docs/configuration/detection-tuning.md` in the guard-core repository for the full field-by-field tuning guide. `PatternCompiler.validate_pattern_safety`'s own probe/wait budget (50ms cost-arbiter soft budget, 2.0s killable-subprocess hard timeout) is a hardcoded constant, not a `SecurityConfig` field; see [the detection reference](detection.md#redos-safety) for the mechanism. -`excluded_detection_headers`, `excluded_detection_params`, `excluded_detection_body_fields` (Headers to exclude from penetration detection scanning. Merged with the hardcoded default exclusion set. / Query parameters to exclude from penetration detection scanning. / JSON body keys to exclude from penetration detection scanning. Matched at any nesting depth, and applied to x-www-form-urlencoded and multipart text-part field names as well.), `detection_anomaly_emission_cooldown` (Min seconds between anomaly events for the same pattern. Raise to reduce noise on low-traffic apps.), `detection_min_samples_for_anomaly` (Minimum samples recorded for a pattern before statistical-anomaly detection engages. Raise to reduce false fires on low-traffic apps.). +`excluded_detection_headers`, `excluded_semantic_detection_headers` (opt-in, default empty; header names, case-insensitive, whose values skip the semantic analyzer only while regex category scanning is unchanged; use `cookie`/`authorization` for apps with JWT sessions; RouteConfig counterpart via `@detection_exclusion(semantic_headers=...)`), `excluded_detection_params`, `excluded_detection_body_fields` (Headers to exclude from penetration detection scanning. Merged with the hardcoded default exclusion set. / Query parameters to exclude from penetration detection scanning. / JSON body keys to exclude from penetration detection scanning. Matched at any nesting depth, and applied to x-www-form-urlencoded and multipart text-part field names as well.), `detection_anomaly_emission_cooldown` (Min seconds between anomaly events for the same pattern. Raise to reduce noise on low-traffic apps.), `detection_min_samples_for_anomaly` (Minimum samples recorded for a pattern before statistical-anomaly detection engages. Raise to reduce false fires on low-traffic apps.). ### Redis diff --git a/guard_core/.agents/skills/guard-core/references/detection.md b/guard_core/.agents/skills/guard-core/references/detection.md index 26a35cf0..30321fc3 100644 --- a/guard_core/.agents/skills/guard-core/references/detection.md +++ b/guard_core/.agents/skills/guard-core/references/detection.md @@ -100,7 +100,7 @@ This validation subprocess mechanism is unrelated to, and isolated from, the liv ## Tuning knobs (SecurityConfig) -`detection_max_content_length` (default 10000), `detection_compiler_timeout` (default 2.0s, range 0.1-10.0), `detection_threat_score_threshold` (default 1.0), `detection_max_scan_values` (default 512, `ge=2`; see [Request-value scan cap](#request-value-scan-cap) above), `enabled_detection_categories` (`frozenset[str]`, subset of the 19, defaults to all 19). `PatternCompiler.validate_pattern_safety`'s 50ms cost-arbiter budget is a hardcoded constant, not a config field; only `max_content_length` (the caller's real content-length cap) is passed in per call. +`detection_max_content_length` (default 10000), `detection_compiler_timeout` (default 2.0s, range 0.1-10.0), `detection_max_scan_seconds` (default 10.0s, range 0.1-60.0; per-request total regex time budget, exhaustion adds a weighted `pattern_timeout` threat `scan_time_budget_exhausted`), `excluded_semantic_detection_headers` (opt-in; header values skip the semantic analyzer only), `detection_threat_score_threshold` (default 1.0), `detection_max_scan_values` (default 512, `ge=2`; see [Request-value scan cap](#request-value-scan-cap) above), `enabled_detection_categories` (`frozenset[str]`, subset of the 19, defaults to all 19). `PatternCompiler.validate_pattern_safety`'s 50ms cost-arbiter budget is a hardcoded constant, not a config field; only `max_content_length` (the caller's real content-length cap) is passed in per call. ## Adding patterns diff --git a/guard_core/_security_config_field_validators.py b/guard_core/_security_config_field_validators.py index ea348a8a..63891f74 100644 --- a/guard_core/_security_config_field_validators.py +++ b/guard_core/_security_config_field_validators.py @@ -465,6 +465,11 @@ def _validate_dynamic_rules_cache_path_value(v: Any) -> Path | None: adapter=_STR_SET_ADAPTER, field_name="excluded_detection_headers", ), + "excluded_semantic_detection_headers": partial( + _validate_typed_collection_field_value, + adapter=_STR_SET_ADAPTER, + field_name="excluded_semantic_detection_headers", + ), "excluded_detection_params": partial( _validate_typed_collection_field_value, adapter=_STR_SET_ADAPTER, diff --git a/guard_core/_security_config_fields.py b/guard_core/_security_config_fields.py index 6891b397..0539c8ff 100644 --- a/guard_core/_security_config_fields.py +++ b/guard_core/_security_config_fields.py @@ -758,6 +758,19 @@ class _SecurityConfigFields(BaseModel): le=10.0, ) + detection_max_scan_seconds: float = Field( + default=10.0, + description=( + "Cap on the total regex scan time (seconds) spent on one request, " + "summed across every field and every pattern. Once reached, " + "remaining patterns and values are not scanned and the request " + "receives a weighted pattern_timeout threat that counts toward the " + "anomaly score, so padding a request cannot skip detection." + ), + ge=0.1, + le=60.0, + ) + detection_pattern_validation_cache_path: str | None = Field( default=None, description=( @@ -1069,6 +1082,16 @@ class _SecurityConfigFields(BaseModel): "the hardcoded default exclusion set." ), ) + excluded_semantic_detection_headers: set[str] = Field( + default_factory=set, + description=( + "Opt-in: header names whose values skip the semantic analyzer " + "only, matched case-insensitively. Regex category scanning of " + "these headers is unchanged. Useful for apps whose JWT session " + "cookies score near detection_semantic_threshold (for example " + "cookie, authorization). Default empty." + ), + ) excluded_detection_params: set[str] = Field( default_factory=set, description=( diff --git a/guard_core/_utils/detection_config.py b/guard_core/_utils/detection_config.py index c0802965..0625e7b2 100644 --- a/guard_core/_utils/detection_config.py +++ b/guard_core/_utils/detection_config.py @@ -232,3 +232,32 @@ def _resolve_binary_min_run_length(config: "SecurityConfig | None") -> int: if config is not None else _DEFAULT_BINARY_MIN_RUN_LENGTH ) + + +_DEFAULT_MAX_SCAN_SECONDS = 10.0 + + +def _resolve_max_scan_seconds(config: "SecurityConfig | None") -> float: + return ( + config.detection_max_scan_seconds + if config is not None + else _DEFAULT_MAX_SCAN_SECONDS + ) + + +def _resolve_semantic_excluded_headers( + config: "SecurityConfig | None", route_config: "RouteConfig | None" +) -> frozenset[str]: + excluded: set[str] = set() + if config is not None: + excluded |= { + h.strip().lower() for h in config.excluded_semantic_detection_headers + } + if ( + route_config is not None + and route_config.excluded_semantic_detection_headers is not None + ): + excluded |= { + h.strip().lower() for h in route_config.excluded_semantic_detection_headers + } + return frozenset(excluded) diff --git a/guard_core/_utils/detection_scan.py b/guard_core/_utils/detection_scan.py index a223621a..f71e05cd 100644 --- a/guard_core/_utils/detection_scan.py +++ b/guard_core/_utils/detection_scan.py @@ -10,6 +10,7 @@ _DEFAULT_BINARY_MIN_RUN_LENGTH, _DEFAULT_MAX_JSON_DEPTH, _DEFAULT_MAX_SCAN_CHARS, + _DEFAULT_MAX_SCAN_SECONDS, _DEFAULT_MAX_SCAN_VALUES, ) from guard_core._utils.logging_utils import _log_at_level, _sanitize_for_reporting @@ -31,6 +32,9 @@ _scan_char_cap_warned: contextvars.ContextVar[bool] = contextvars.ContextVar( "guard_core_detection_scan_char_cap_warned", default=False ) +_scan_time_cap_warned: contextvars.ContextVar[bool] = contextvars.ContextVar( + "guard_core_detection_scan_time_cap_warned", default=False +) _json_depth_cap: contextvars.ContextVar[int] = contextvars.ContextVar( "guard_core_detection_json_depth_cap", default=_DEFAULT_MAX_JSON_DEPTH ) @@ -40,6 +44,17 @@ _binary_min_run: contextvars.ContextVar[int] = contextvars.ContextVar( "guard_core_detection_binary_min_run", default=_DEFAULT_BINARY_MIN_RUN_LENGTH ) +_scan_time_cap: contextvars.ContextVar[float | None] = contextvars.ContextVar( + "guard_core_detection_scan_time_cap", default=None +) +_scan_time_spent: contextvars.ContextVar[float] = contextvars.ContextVar( + "guard_core_detection_scan_time_spent", default=0.0 +) +_semantic_excluded_headers: contextvars.ContextVar[frozenset[str]] = ( + contextvars.ContextVar( + "guard_core_detection_semantic_excluded_headers", default=frozenset() + ) +) @contextmanager @@ -48,6 +63,8 @@ def _scan_value_budget( max_json_depth: int = _DEFAULT_MAX_JSON_DEPTH, max_scan_chars: int = _DEFAULT_MAX_SCAN_CHARS, binary_min_run_length: int = _DEFAULT_BINARY_MIN_RUN_LENGTH, + max_scan_seconds: float = _DEFAULT_MAX_SCAN_SECONDS, + semantic_excluded_headers: frozenset[str] = frozenset(), ) -> Iterator[None]: count_token = _scanned_value_count.set(0) cap_token = _scan_value_cap.set(max_values) @@ -57,6 +74,10 @@ def _scan_value_budget( depth_cap_token = _json_depth_cap.set(max_json_depth) depth_warned_token = _json_depth_warned.set(False) binary_run_token = _binary_min_run.set(binary_min_run_length) + time_cap_token = _scan_time_cap.set(max_scan_seconds) + time_spent_token = _scan_time_spent.set(0.0) + time_warned_token = _scan_time_cap_warned.set(False) + semantic_token = _semantic_excluded_headers.set(semantic_excluded_headers) try: yield finally: @@ -68,12 +89,77 @@ def _scan_value_budget( _json_depth_cap.reset(depth_cap_token) _json_depth_warned.reset(depth_warned_token) _binary_min_run.reset(binary_run_token) + _scan_time_cap.reset(time_cap_token) + _scan_time_spent.reset(time_spent_token) + _scan_time_cap_warned.reset(time_warned_token) + _semantic_excluded_headers.reset(semantic_token) def _binary_island_min_run_length() -> int: return _binary_min_run.get() +def _scan_time_remaining() -> float | None: + cap = _scan_time_cap.get() + return None if cap is None else cap - _scan_time_spent.get() + + +_SCAN_TIME_BUDGET_EXHAUSTED_PATTERN = "scan_time_budget_exhausted" +_MIN_CLAMPED_TIMEOUT = 0.001 +_EVENT_LOOP_SLICE_SECONDS = 0.002 + + +def _scan_time_budget_exhausted_threat(weight: float) -> dict[str, Any]: + return { + "type": "pattern_timeout", + "pattern": _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, + "match": "", + "position": 0, + "execution_time": 0.0, + "category": "custom", + "weight": weight, + } + + +def _clamp_to_scan_time_budget(timeout: float) -> float: + remaining = _scan_time_remaining() + if remaining is None: + return timeout + return max(min(timeout, remaining), _MIN_CLAMPED_TIMEOUT) + + +def _record_scan_time(elapsed: float) -> None: + _scan_time_spent.set(_scan_time_spent.get() + elapsed) + + +def _scan_time_budget_exhausted() -> bool: + remaining = _scan_time_remaining() + return remaining is not None and remaining <= 0.0 + + +def _warn_scan_time_cap_reached_once(client_ip: str) -> None: + if _scan_time_cap_warned.get(): + return + _scan_time_cap_warned.set(True) + logger.warning( + "detection_max_scan_seconds (%s) reached for client %s; remaining " + "patterns and request values are not scanned", + _scan_time_cap.get(), + client_ip, + ) + + +def _context_header_is_semantic_excluded(context: str) -> bool: + excluded = _semantic_excluded_headers.get() + if not excluded: + return False + kind, _, rest = context.partition(":") + if kind != "header": + return False + name = rest.removesuffix(":embedded_json") + return name.strip().lower() in excluded + + def _scan_value_budget_exhausted(client_ip: str) -> bool: count = _scanned_value_count.get() + 1 _scanned_value_count.set(count) @@ -218,6 +304,8 @@ async def _fallback_pattern_check( all_compiled = await sus_patterns_handler.get_all_compiled_patterns() binary_prefix = build_binary_prefix(value) for pattern, _contexts, category in all_compiled: + if _scan_time_budget_exhausted(): + return True, "Scan time budget exhausted (fallback)" pattern_start = time.monotonic() try: threat, _timeout_occurred = await sus_patterns_handler._check_regex_pattern( diff --git a/guard_core/_utils/penetration_detection.py b/guard_core/_utils/penetration_detection.py index 415888bb..65868787 100644 --- a/guard_core/_utils/penetration_detection.py +++ b/guard_core/_utils/penetration_detection.py @@ -15,8 +15,10 @@ _resolve_log_level, _resolve_max_json_depth, _resolve_max_scan_chars, + _resolve_max_scan_seconds, _resolve_max_scan_values, _resolve_scan_body, + _resolve_semantic_excluded_headers, _resolve_sensitive_log_body_fields, _resolve_sensitive_log_headers, _resolve_sensitive_log_params, @@ -190,9 +192,16 @@ async def detect_penetration_attempt( max_json_depth = _resolve_max_json_depth(config) max_scan_chars = _resolve_max_scan_chars(config) binary_min_run = _resolve_binary_min_run_length(config) + max_scan_seconds = _resolve_max_scan_seconds(config) + semantic_excluded_headers = _resolve_semantic_excluded_headers(config, route_config) with _scan_value_budget( - max_scan_values, max_json_depth, max_scan_chars, binary_min_run + max_scan_values, + max_json_depth, + max_scan_chars, + binary_min_run, + max_scan_seconds, + semantic_excluded_headers, ): surface_hit = await _scan_request_surface( request, diff --git a/guard_core/decorators/content_filtering.py b/guard_core/decorators/content_filtering.py index 13a84248..10e8b043 100644 --- a/guard_core/decorators/content_filtering.py +++ b/guard_core/decorators/content_filtering.py @@ -84,11 +84,14 @@ def detection_exclusion( body_fields: set[str] | None = None, categories: set[str] | None = None, scan_body: bool | None = None, + semantic_headers: set[str] | None = None, ) -> Callable[[Callable[..., Any]], DecoratedFunction]: def decorator(func: Callable[..., Any]) -> DecoratedFunction: route_config = self._ensure_route_config(func) if headers is not None: route_config.excluded_detection_headers = set(headers) + if semantic_headers is not None: + route_config.excluded_semantic_detection_headers = set(semantic_headers) if params is not None: route_config.excluded_detection_params = set(params) if body_fields is not None: diff --git a/guard_core/decorators/route_config.py b/guard_core/decorators/route_config.py index 3604ab2b..773ed17a 100644 --- a/guard_core/decorators/route_config.py +++ b/guard_core/decorators/route_config.py @@ -271,6 +271,7 @@ def __init__(self, revision: RouteConfigRevision | None = None) -> None: self.authorization_header_required: str | None = None self.geo_rate_limits: dict[str, tuple[int, int]] | None = None self.excluded_detection_headers: set[str] | None = None + self.excluded_semantic_detection_headers: set[str] | None = None self.excluded_detection_params: set[str] | None = None self.excluded_detection_body_fields: set[str] | None = None self.enabled_detection_categories: set[str] | None = None diff --git a/guard_core/detection_engine/compiler.py b/guard_core/detection_engine/compiler.py index d02422ed..d7e784f0 100644 --- a/guard_core/detection_engine/compiler.py +++ b/guard_core/detection_engine/compiler.py @@ -4,6 +4,7 @@ import re import threading from collections.abc import Awaitable, Callable +from typing import Any from guard_core.detection_engine import _validation_cache from guard_core.detection_engine._redos_cost_arbiter import ( @@ -38,6 +39,19 @@ def shared_regex_executor() -> concurrent.futures.ThreadPoolExecutor: return _shared_executor +async def _await_regex_future( + future: concurrent.futures.Future[Any], timeout: float +) -> Any: + try: + return await asyncio.wait_for(asyncio.wrap_future(future), timeout=timeout) + except asyncio.TimeoutError as exc: + raise concurrent.futures.TimeoutError() from exc + + +async def _yield_to_event_loop() -> None: + await asyncio.sleep(0) + + def report_scan_success() -> None: global _consecutive_timeouts if _consecutive_timeouts: diff --git a/guard_core/handlers/_suspatterns_regex.py b/guard_core/handlers/_suspatterns_regex.py index 0aa244eb..e5ced6bd 100644 --- a/guard_core/handlers/_suspatterns_regex.py +++ b/guard_core/handlers/_suspatterns_regex.py @@ -5,12 +5,23 @@ from collections.abc import Callable, Iterator from typing import Any -from guard_core._utils.detection_scan import _redact_pattern_source +from guard_core._utils.detection_scan import ( + _EVENT_LOOP_SLICE_SECONDS, + _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, + _clamp_to_scan_time_budget, + _record_scan_time, + _redact_pattern_source, + _scan_time_budget_exhausted, + _scan_time_budget_exhausted_threat, + _warn_scan_time_cap_reached_once, +) from guard_core.detection_engine.binary_prefix import ( build_binary_prefix, match_is_binary_dense, ) from guard_core.detection_engine.compiler import ( + _await_regex_future, + _yield_to_event_loop, report_scan_success, report_scan_timeout, shared_regex_executor, @@ -453,9 +464,10 @@ async def _check_windowed_pattern( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) + timeout = _clamp_to_scan_time_budget(timeout) future = shared_regex_executor().submit(lambda: list(finder(content))) try: - matches = future.result(timeout=timeout) + matches = await _await_regex_future(future, timeout) report_scan_success() except concurrent.futures.TimeoutError: logger.warning( @@ -520,9 +532,10 @@ async def _check_pattern_with_timeout( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) + timeout = _clamp_to_scan_time_budget(timeout) future = shared_regex_executor().submit(pattern.search, content, search_from) try: - match = future.result(timeout=timeout) + match = await _await_regex_future(future, timeout) report_scan_success() return match, False except concurrent.futures.TimeoutError: @@ -582,7 +595,19 @@ async def _check_regex_patterns( performance_monitor = state.performance_monitor binary_prefix = build_binary_prefix(content) + slice_start = time.monotonic() for pattern, contexts, category in all_patterns: + if _scan_time_budget_exhausted(): + budget_threat = _scan_time_budget_exhausted_threat( + _resolve_pattern_weight( + _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, "custom" + ) + ) + _warn_scan_time_cap_reached_once(ip_address) + threats.append(budget_threat) + matched_patterns.append(budget_threat["pattern"]) + timeouts.append(budget_threat["pattern"]) + break if _pattern_should_be_skipped( pattern, contexts, @@ -609,6 +634,12 @@ async def _check_regex_patterns( binary_prefix=binary_prefix, ) + pattern_end = time.monotonic() + _record_scan_time(pattern_end - pattern_start) + if pattern_end - slice_start >= _EVENT_LOOP_SLICE_SECONDS: + await _yield_to_event_loop() + slice_start = time.monotonic() + if timeout_occurred: timeouts.append(pattern.pattern) if not threat: diff --git a/guard_core/handlers/suspatterns_handler.py b/guard_core/handlers/suspatterns_handler.py index 4752acad..8413a2c7 100644 --- a/guard_core/handlers/suspatterns_handler.py +++ b/guard_core/handlers/suspatterns_handler.py @@ -16,6 +16,7 @@ from typing import Any from typing import NamedTuple as NamedTuple +from guard_core._utils.detection_scan import _context_header_is_semantic_excluded from guard_core.detection_engine import ( ContentPreprocessor, PatternCompiler, @@ -932,9 +933,11 @@ async def detect( matched_patterns = matched_patterns + short_base64_matched timeouts = timeouts + short_base64_timeouts - semantic_threats, semantic_score = await self._check_semantic_threats( - processed_content, state=state, raw_content=original_content - ) + semantic_threats: list[dict] = [] + if not _context_header_is_semantic_excluded(context): + semantic_threats, _semantic_score = await self._check_semantic_threats( + processed_content, state=state, raw_content=original_content + ) threats = regex_threats + semantic_threats is_threat = ( diff --git a/guard_core/sync/_utils/detection_config.py b/guard_core/sync/_utils/detection_config.py index 924ab0b3..b34dfb29 100644 --- a/guard_core/sync/_utils/detection_config.py +++ b/guard_core/sync/_utils/detection_config.py @@ -232,3 +232,32 @@ def _resolve_binary_min_run_length(config: "SecurityConfig | None") -> int: if config is not None else _DEFAULT_BINARY_MIN_RUN_LENGTH ) + + +_DEFAULT_MAX_SCAN_SECONDS = 10.0 + + +def _resolve_max_scan_seconds(config: "SecurityConfig | None") -> float: + return ( + config.detection_max_scan_seconds + if config is not None + else _DEFAULT_MAX_SCAN_SECONDS + ) + + +def _resolve_semantic_excluded_headers( + config: "SecurityConfig | None", route_config: "RouteConfig | None" +) -> frozenset[str]: + excluded: set[str] = set() + if config is not None: + excluded |= { + h.strip().lower() for h in config.excluded_semantic_detection_headers + } + if ( + route_config is not None + and route_config.excluded_semantic_detection_headers is not None + ): + excluded |= { + h.strip().lower() for h in route_config.excluded_semantic_detection_headers + } + return frozenset(excluded) diff --git a/guard_core/sync/_utils/detection_scan.py b/guard_core/sync/_utils/detection_scan.py index e511c9d8..28d213f5 100644 --- a/guard_core/sync/_utils/detection_scan.py +++ b/guard_core/sync/_utils/detection_scan.py @@ -10,6 +10,7 @@ _DEFAULT_BINARY_MIN_RUN_LENGTH, _DEFAULT_MAX_JSON_DEPTH, _DEFAULT_MAX_SCAN_CHARS, + _DEFAULT_MAX_SCAN_SECONDS, _DEFAULT_MAX_SCAN_VALUES, ) from guard_core.sync._utils.logging_utils import _log_at_level, _sanitize_for_reporting @@ -31,6 +32,9 @@ _scan_char_cap_warned: contextvars.ContextVar[bool] = contextvars.ContextVar( "guard_core_detection_scan_char_cap_warned", default=False ) +_scan_time_cap_warned: contextvars.ContextVar[bool] = contextvars.ContextVar( + "guard_core_detection_scan_time_cap_warned", default=False +) _json_depth_cap: contextvars.ContextVar[int] = contextvars.ContextVar( "guard_core_detection_json_depth_cap", default=_DEFAULT_MAX_JSON_DEPTH ) @@ -40,6 +44,17 @@ _binary_min_run: contextvars.ContextVar[int] = contextvars.ContextVar( "guard_core_detection_binary_min_run", default=_DEFAULT_BINARY_MIN_RUN_LENGTH ) +_scan_time_cap: contextvars.ContextVar[float | None] = contextvars.ContextVar( + "guard_core_detection_scan_time_cap", default=None +) +_scan_time_spent: contextvars.ContextVar[float] = contextvars.ContextVar( + "guard_core_detection_scan_time_spent", default=0.0 +) +_semantic_excluded_headers: contextvars.ContextVar[frozenset[str]] = ( + contextvars.ContextVar( + "guard_core_detection_semantic_excluded_headers", default=frozenset() + ) +) @contextmanager @@ -48,6 +63,8 @@ def _scan_value_budget( max_json_depth: int = _DEFAULT_MAX_JSON_DEPTH, max_scan_chars: int = _DEFAULT_MAX_SCAN_CHARS, binary_min_run_length: int = _DEFAULT_BINARY_MIN_RUN_LENGTH, + max_scan_seconds: float = _DEFAULT_MAX_SCAN_SECONDS, + semantic_excluded_headers: frozenset[str] = frozenset(), ) -> Iterator[None]: count_token = _scanned_value_count.set(0) cap_token = _scan_value_cap.set(max_values) @@ -57,6 +74,10 @@ def _scan_value_budget( depth_cap_token = _json_depth_cap.set(max_json_depth) depth_warned_token = _json_depth_warned.set(False) binary_run_token = _binary_min_run.set(binary_min_run_length) + time_cap_token = _scan_time_cap.set(max_scan_seconds) + time_spent_token = _scan_time_spent.set(0.0) + time_warned_token = _scan_time_cap_warned.set(False) + semantic_token = _semantic_excluded_headers.set(semantic_excluded_headers) try: yield finally: @@ -68,12 +89,77 @@ def _scan_value_budget( _json_depth_cap.reset(depth_cap_token) _json_depth_warned.reset(depth_warned_token) _binary_min_run.reset(binary_run_token) + _scan_time_cap.reset(time_cap_token) + _scan_time_spent.reset(time_spent_token) + _scan_time_cap_warned.reset(time_warned_token) + _semantic_excluded_headers.reset(semantic_token) def _binary_island_min_run_length() -> int: return _binary_min_run.get() +def _scan_time_remaining() -> float | None: + cap = _scan_time_cap.get() + return None if cap is None else cap - _scan_time_spent.get() + + +_SCAN_TIME_BUDGET_EXHAUSTED_PATTERN = "scan_time_budget_exhausted" +_MIN_CLAMPED_TIMEOUT = 0.001 +_EVENT_LOOP_SLICE_SECONDS = 0.002 + + +def _scan_time_budget_exhausted_threat(weight: float) -> dict[str, Any]: + return { + "type": "pattern_timeout", + "pattern": _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, + "match": "", + "position": 0, + "execution_time": 0.0, + "category": "custom", + "weight": weight, + } + + +def _clamp_to_scan_time_budget(timeout: float) -> float: + remaining = _scan_time_remaining() + if remaining is None: + return timeout + return max(min(timeout, remaining), _MIN_CLAMPED_TIMEOUT) + + +def _record_scan_time(elapsed: float) -> None: + _scan_time_spent.set(_scan_time_spent.get() + elapsed) + + +def _scan_time_budget_exhausted() -> bool: + remaining = _scan_time_remaining() + return remaining is not None and remaining <= 0.0 + + +def _warn_scan_time_cap_reached_once(client_ip: str) -> None: + if _scan_time_cap_warned.get(): + return + _scan_time_cap_warned.set(True) + logger.warning( + "detection_max_scan_seconds (%s) reached for client %s; remaining " + "patterns and request values are not scanned", + _scan_time_cap.get(), + client_ip, + ) + + +def _context_header_is_semantic_excluded(context: str) -> bool: + excluded = _semantic_excluded_headers.get() + if not excluded: + return False + kind, _, rest = context.partition(":") + if kind != "header": + return False + name = rest.removesuffix(":embedded_json") + return name.strip().lower() in excluded + + def _scan_value_budget_exhausted(client_ip: str) -> bool: count = _scanned_value_count.get() + 1 _scanned_value_count.set(count) @@ -216,6 +302,8 @@ def _fallback_pattern_check( all_compiled = sus_patterns_handler.get_all_compiled_patterns() binary_prefix = build_binary_prefix(value) for pattern, _contexts, category in all_compiled: + if _scan_time_budget_exhausted(): + return True, "Scan time budget exhausted (fallback)" pattern_start = time.monotonic() try: threat, _timeout_occurred = sus_patterns_handler._check_regex_pattern( diff --git a/guard_core/sync/_utils/penetration_detection.py b/guard_core/sync/_utils/penetration_detection.py index 7e33bb87..4e1603ff 100644 --- a/guard_core/sync/_utils/penetration_detection.py +++ b/guard_core/sync/_utils/penetration_detection.py @@ -15,8 +15,10 @@ _resolve_log_level, _resolve_max_json_depth, _resolve_max_scan_chars, + _resolve_max_scan_seconds, _resolve_max_scan_values, _resolve_scan_body, + _resolve_semantic_excluded_headers, _resolve_sensitive_log_body_fields, _resolve_sensitive_log_headers, _resolve_sensitive_log_params, @@ -190,9 +192,16 @@ def detect_penetration_attempt( max_json_depth = _resolve_max_json_depth(config) max_scan_chars = _resolve_max_scan_chars(config) binary_min_run = _resolve_binary_min_run_length(config) + max_scan_seconds = _resolve_max_scan_seconds(config) + semantic_excluded_headers = _resolve_semantic_excluded_headers(config, route_config) with _scan_value_budget( - max_scan_values, max_json_depth, max_scan_chars, binary_min_run + max_scan_values, + max_json_depth, + max_scan_chars, + binary_min_run, + max_scan_seconds, + semantic_excluded_headers, ): surface_hit = _scan_request_surface( request, diff --git a/guard_core/sync/decorators/content_filtering.py b/guard_core/sync/decorators/content_filtering.py index 53df6e16..fffa3807 100644 --- a/guard_core/sync/decorators/content_filtering.py +++ b/guard_core/sync/decorators/content_filtering.py @@ -84,11 +84,14 @@ def detection_exclusion( body_fields: set[str] | None = None, categories: set[str] | None = None, scan_body: bool | None = None, + semantic_headers: set[str] | None = None, ) -> Callable[[Callable[..., Any]], DecoratedFunction]: def decorator(func: Callable[..., Any]) -> DecoratedFunction: route_config = self._ensure_route_config(func) if headers is not None: route_config.excluded_detection_headers = set(headers) + if semantic_headers is not None: + route_config.excluded_semantic_detection_headers = set(semantic_headers) if params is not None: route_config.excluded_detection_params = set(params) if body_fields is not None: diff --git a/guard_core/sync/decorators/route_config.py b/guard_core/sync/decorators/route_config.py index 9c3431ba..4feb752b 100644 --- a/guard_core/sync/decorators/route_config.py +++ b/guard_core/sync/decorators/route_config.py @@ -271,6 +271,7 @@ def __init__(self, revision: RouteConfigRevision | None = None) -> None: self.authorization_header_required: str | None = None self.geo_rate_limits: dict[str, tuple[int, int]] | None = None self.excluded_detection_headers: set[str] | None = None + self.excluded_semantic_detection_headers: set[str] | None = None self.excluded_detection_params: set[str] | None = None self.excluded_detection_body_fields: set[str] | None = None self.enabled_detection_categories: set[str] | None = None diff --git a/guard_core/sync/detection_engine/compiler.py b/guard_core/sync/detection_engine/compiler.py index c99c4a8a..64bd0455 100644 --- a/guard_core/sync/detection_engine/compiler.py +++ b/guard_core/sync/detection_engine/compiler.py @@ -3,6 +3,7 @@ import re import threading from collections.abc import Callable +from typing import Any from guard_core.sync.detection_engine import _validation_cache from guard_core.sync.detection_engine._redos_cost_arbiter import ( @@ -37,6 +38,14 @@ def shared_regex_executor() -> concurrent.futures.ThreadPoolExecutor: return _shared_executor +def _await_regex_future(future: concurrent.futures.Future[Any], timeout: float) -> Any: + return future.result(timeout=timeout) + + +def _yield_to_event_loop() -> None: + return None + + def report_scan_success() -> None: global _consecutive_timeouts if _consecutive_timeouts: diff --git a/guard_core/sync/handlers/_suspatterns_regex.py b/guard_core/sync/handlers/_suspatterns_regex.py index 1a82dec9..95eed1c7 100644 --- a/guard_core/sync/handlers/_suspatterns_regex.py +++ b/guard_core/sync/handlers/_suspatterns_regex.py @@ -5,12 +5,23 @@ from collections.abc import Callable, Iterator from typing import Any -from guard_core.sync._utils.detection_scan import _redact_pattern_source +from guard_core.sync._utils.detection_scan import ( + _EVENT_LOOP_SLICE_SECONDS, + _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, + _clamp_to_scan_time_budget, + _record_scan_time, + _redact_pattern_source, + _scan_time_budget_exhausted, + _scan_time_budget_exhausted_threat, + _warn_scan_time_cap_reached_once, +) from guard_core.sync.detection_engine.binary_prefix import ( build_binary_prefix, match_is_binary_dense, ) from guard_core.sync.detection_engine.compiler import ( + _await_regex_future, + _yield_to_event_loop, report_scan_success, report_scan_timeout, shared_regex_executor, @@ -453,9 +464,10 @@ def _check_windowed_pattern( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) + timeout = _clamp_to_scan_time_budget(timeout) future = shared_regex_executor().submit(lambda: list(finder(content))) try: - matches = future.result(timeout=timeout) + matches = _await_regex_future(future, timeout) report_scan_success() except concurrent.futures.TimeoutError: logger.warning( @@ -520,9 +532,10 @@ def _check_pattern_with_timeout( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) + timeout = _clamp_to_scan_time_budget(timeout) future = shared_regex_executor().submit(pattern.search, content, search_from) try: - match = future.result(timeout=timeout) + match = _await_regex_future(future, timeout) report_scan_success() return match, False except concurrent.futures.TimeoutError: @@ -582,7 +595,19 @@ def _check_regex_patterns( performance_monitor = state.performance_monitor binary_prefix = build_binary_prefix(content) + slice_start = time.monotonic() for pattern, contexts, category in all_patterns: + if _scan_time_budget_exhausted(): + budget_threat = _scan_time_budget_exhausted_threat( + _resolve_pattern_weight( + _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, "custom" + ) + ) + _warn_scan_time_cap_reached_once(ip_address) + threats.append(budget_threat) + matched_patterns.append(budget_threat["pattern"]) + timeouts.append(budget_threat["pattern"]) + break if _pattern_should_be_skipped( pattern, contexts, @@ -609,6 +634,12 @@ def _check_regex_patterns( binary_prefix=binary_prefix, ) + pattern_end = time.monotonic() + _record_scan_time(pattern_end - pattern_start) + if pattern_end - slice_start >= _EVENT_LOOP_SLICE_SECONDS: + _yield_to_event_loop() + slice_start = time.monotonic() + if timeout_occurred: timeouts.append(pattern.pattern) if not threat: diff --git a/guard_core/sync/handlers/suspatterns_handler.py b/guard_core/sync/handlers/suspatterns_handler.py index 4a05d89a..1aada41e 100644 --- a/guard_core/sync/handlers/suspatterns_handler.py +++ b/guard_core/sync/handlers/suspatterns_handler.py @@ -16,6 +16,7 @@ from typing import Any from typing import NamedTuple as NamedTuple +from guard_core.sync._utils.detection_scan import _context_header_is_semantic_excluded from guard_core.sync.detection_engine import ( ContentPreprocessor, PatternCompiler, @@ -932,9 +933,11 @@ def detect( matched_patterns = matched_patterns + short_base64_matched timeouts = timeouts + short_base64_timeouts - semantic_threats, semantic_score = self._check_semantic_threats( - processed_content, state=state, raw_content=original_content - ) + semantic_threats: list[dict] = [] + if not _context_header_is_semantic_excluded(context): + semantic_threats, _semantic_score = self._check_semantic_threats( + processed_content, state=state, raw_content=original_content + ) threats = regex_threats + semantic_threats is_threat = ( diff --git a/scripts/unasync.py b/scripts/unasync.py index 553bb131..e83c4216 100644 --- a/scripts/unasync.py +++ b/scripts/unasync.py @@ -27,6 +27,7 @@ } SKIP_TESTS = { + "test_scan_event_loop_yield.py", "test_module_imports_standalone.py", "test_suspatterns_handler_module_surface.py", "test_gc_per_test_opt_in.py", @@ -146,7 +147,36 @@ " return result\n" ) +_ASYNC_AWAIT_REGEX_FUTURE_SRC = ( + "async def _await_regex_future(\n" + " future: concurrent.futures.Future[Any], timeout: float\n" + ") -> Any:\n" + " try:\n" + " return await asyncio.wait_for(" + "asyncio.wrap_future(future), timeout=timeout)\n" + " except asyncio.TimeoutError as exc:\n" + " raise concurrent.futures.TimeoutError() from exc\n" +) + +_SYNC_AWAIT_REGEX_FUTURE_REPLACEMENT = ( + "def _await_regex_future(\n" + " future: concurrent.futures.Future[Any], timeout: float\n" + ") -> Any:\n" + " return future.result(timeout=timeout)\n" +) + +_ASYNC_YIELD_SRC = ( + "async def _yield_to_event_loop() -> None:\n await asyncio.sleep(0)\n" +) + +_SYNC_YIELD_REPLACEMENT = "def _yield_to_event_loop() -> None:\n return None\n" + SUBS: list[tuple[str, str]] = [ + (re.escape(_ASYNC_YIELD_SRC), _SYNC_YIELD_REPLACEMENT), + ( + re.escape(_ASYNC_AWAIT_REGEX_FUTURE_SRC), + _SYNC_AWAIT_REGEX_FUTURE_REPLACEMENT.replace("\\", "\\\\"), + ), ( re.escape(_ASYNC_SAFE_READ_SRC), _SYNC_SAFE_READ_REPLACEMENT.replace("\\", "\\\\"), diff --git a/tests/test_decorators/test_detection_exclusion.py b/tests/test_decorators/test_detection_exclusion.py index 2a539c09..d7820af2 100644 --- a/tests/test_decorators/test_detection_exclusion.py +++ b/tests/test_decorators/test_detection_exclusion.py @@ -100,3 +100,16 @@ def test_detection_exclusion_no_args_creates_route_config() -> None: assert rc.excluded_detection_params is None assert rc.excluded_detection_body_fields is None assert rc.enabled_detection_categories is None + + +def test_detection_exclusion_sets_semantic_headers() -> None: + d = _FakeDecorator(SecurityConfig()) + decorated = _decorate(d.detection_exclusion(semantic_headers={"cookie"})) + rc = d.get_route_config(d._get_route_id(decorated)) + assert rc is not None + assert rc.excluded_semantic_detection_headers == {"cookie"} + assert rc.excluded_detection_headers is None + + +def test_semantic_headers_default_to_none() -> None: + assert RouteConfig().excluded_semantic_detection_headers is None diff --git a/tests/test_models/test_every_collection_field_is_revalidated.py b/tests/test_models/test_every_collection_field_is_revalidated.py index 35b6bfc9..d37e561f 100644 --- a/tests/test_models/test_every_collection_field_is_revalidated.py +++ b/tests/test_models/test_every_collection_field_is_revalidated.py @@ -65,6 +65,7 @@ def test_collection_field_is_revalidated_or_immutable(field_name: str) -> None: ("agent_sensitive_headers", ["authorization"], "nope"), ("otel_resource_attributes", {"service.name": "guard"}, "nope"), ("excluded_detection_headers", {"x-foo"}, "nope"), + ("excluded_semantic_detection_headers", {"cookie"}, "nope"), ("excluded_detection_params", {"token"}, "nope"), ("excluded_detection_body_fields", {"password"}, "nope"), ) diff --git a/tests/test_scan_event_loop_yield.py b/tests/test_scan_event_loop_yield.py new file mode 100644 index 00000000..9d933880 --- /dev/null +++ b/tests/test_scan_event_loop_yield.py @@ -0,0 +1,110 @@ +import asyncio +import concurrent.futures +import threading +from collections.abc import AsyncGenerator + +import pytest + +from guard_core.detection_engine import compiler as compiler_module +from guard_core.detection_engine.compiler import _await_regex_future +from guard_core.handlers._suspatterns_regex import _WINDOWED_PATTERN_FINDERS +from guard_core.handlers.suspatterns_handler import SusPatternsManager +from guard_core.models import SecurityConfig + + +@pytest.fixture(autouse=True) +async def _reset_suspatterns_singleton() -> AsyncGenerator[None, None]: + original_instance = SusPatternsManager._instance + original_config = SusPatternsManager._config + + SusPatternsManager._instance = None + SusPatternsManager._config = None + + yield + + SusPatternsManager._instance = original_instance + SusPatternsManager._config = original_config + + +async def test_await_regex_future_returns_result_and_yields_to_loop() -> None: + ticks: list[int] = [] + + async def ticker() -> None: + for i in range(3): + ticks.append(i) + await asyncio.sleep(0) + + task = asyncio.create_task(ticker()) + future = compiler_module.shared_regex_executor().submit(lambda: "done") + + assert await _await_regex_future(future, 5.0) == "done" + await task + assert ticks == [0, 1, 2] + + +async def test_await_regex_future_timeout_is_concurrent_timeout() -> None: + release = threading.Event() + future = compiler_module.shared_regex_executor().submit(release.wait, 5.0) + try: + with pytest.raises(concurrent.futures.TimeoutError): + await _await_regex_future(future, 0.01) + finally: + release.set() + + +async def test_other_tasks_run_between_patterns_during_a_scan() -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + patterns = await manager.get_all_compiled_patterns() + windowed = [ + p for p, _ctx, _cat in patterns if p.pattern in _WINDOWED_PATTERN_FINDERS + ] + assert windowed + + ran: list[str] = [] + + async def other_request() -> None: + ran.append("other") + + other = asyncio.create_task(other_request()) + start_count = len(ran) + + await manager._check_regex_pattern( + windowed[0], "hello world", "127.0.0.1", 0.0, "sqli" + ) + + assert start_count == 0 + assert ran == ["other"] + await other + + +async def test_pattern_loop_yields_to_other_tasks_once_a_slice_elapses() -> None: + from unittest.mock import patch + + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + ticks = iter(x * 0.01 for x in range(1_000_000)) + ran_at: list[int] = [] + progress = {"patterns": 0} + real = manager._check_regex_pattern + + async def counting(*args: object, **kwargs: object) -> object: + progress["patterns"] += 1 + return await real(*args, **kwargs) # type: ignore[arg-type] + + async def other_request() -> None: + ran_at.append(progress["patterns"]) + + other = asyncio.create_task(other_request()) + with ( + patch("guard_core.handlers._suspatterns_regex.time") as fake_time, + patch.object(manager, "_check_regex_pattern", counting), + ): + fake_time.monotonic = lambda: next(ticks) + await manager._check_regex_patterns("hello world", "127.0.0.1", None) + + await other + all_patterns = await manager.get_all_compiled_patterns() + assert ran_at + assert ran_at[0] < len(all_patterns) + assert ran_at[0] <= 2 diff --git a/tests/test_scan_time_budget.py b/tests/test_scan_time_budget.py new file mode 100644 index 00000000..fea95623 --- /dev/null +++ b/tests/test_scan_time_budget.py @@ -0,0 +1,153 @@ +from collections.abc import AsyncGenerator, Iterator +from typing import Any +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +from guard_core._utils import detection_scan +from guard_core._utils.detection_scan import ( + _scan_time_budget_exhausted, + _scan_value_budget, +) +from guard_core.handlers.suspatterns_handler import SusPatternsManager +from guard_core.models import SecurityConfig +from guard_core.utils import detect_penetration_attempt +from tests.conftest import MockGuardRequest + + +@pytest.fixture(autouse=True) +async def _reset_suspatterns_singleton() -> AsyncGenerator[None, None]: + original_instance = SusPatternsManager._instance + original_config = SusPatternsManager._config + + SusPatternsManager._instance = None + SusPatternsManager._config = None + + yield + + SusPatternsManager._instance = original_instance + SusPatternsManager._config = original_config + + +class _FakeClock: + def __init__(self, step: float) -> None: + self.now = 0.0 + self.step = step + + def monotonic(self) -> float: + self.now += self.step + return self.now + + +@pytest.fixture +def fake_regex_clock() -> Iterator[_FakeClock]: + clock = _FakeClock(step=1.0) + with patch("guard_core.handlers._suspatterns_regex.time") as fake_time: + fake_time.monotonic = clock.monotonic + yield clock + + +def test_field_default_and_bounds() -> None: + assert SecurityConfig().detection_max_scan_seconds == 10.0 + assert SecurityConfig(detection_max_scan_seconds=0.1) + assert SecurityConfig(detection_max_scan_seconds=60.0) + for bad in (0.0, 0.09, 60.1): + with pytest.raises(ValidationError): + SecurityConfig(detection_max_scan_seconds=bad) + + +def test_budget_is_unbounded_outside_a_request() -> None: + assert _scan_time_budget_exhausted() is False + + +def test_budget_tracks_recorded_time_and_resets() -> None: + with _scan_value_budget(512, max_scan_seconds=1.0): + assert _scan_time_budget_exhausted() is False + detection_scan._record_scan_time(0.4) + assert _scan_time_budget_exhausted() is False + detection_scan._record_scan_time(0.6) + assert _scan_time_budget_exhausted() is True + assert _scan_time_budget_exhausted() is False + with _scan_value_budget(512, max_scan_seconds=1.0): + assert _scan_time_budget_exhausted() is False + + +async def test_default_budget_does_not_flag_benign_request() -> None: + request = MockGuardRequest( + path="/items?page=2", headers={"accept-language": "en-US,en;q=0.9"} + ) + + result = await detect_penetration_attempt(request, SecurityConfig()) + + assert result.is_threat is False + + +async def test_exhausted_budget_stops_scan_and_flags_request( + fake_regex_clock: _FakeClock, +) -> None: + request = MockGuardRequest(path="/items", headers={"x-pad": "benign filler value"}) + manager = SusPatternsManager() + seen: list[str] = [] + real = manager._check_regex_pattern + + async def counting(pattern: Any, *args: Any, **kwargs: Any) -> Any: + seen.append(pattern.pattern) + return await real(pattern, *args, **kwargs) + + with patch.object(manager, "_check_regex_pattern", counting): + result = await detect_penetration_attempt( + request, SecurityConfig(detection_max_scan_seconds=0.1) + ) + + all_patterns = await manager.get_all_compiled_patterns() + assert result.is_threat is True + assert "Pattern exceeded scan time budget" in result.trigger_info + assert len(seen) < len(all_patterns) + assert fake_regex_clock.now > 0 + + +async def test_budget_threat_is_weighted_and_reported_as_timeout( + fake_regex_clock: _FakeClock, +) -> None: + config = SecurityConfig(detection_max_scan_seconds=0.1) + manager = SusPatternsManager() + manager.configure(config) + + with _scan_value_budget(512, max_scan_seconds=0.1): + result = await manager.detect("hello world", "127.0.0.1", "query_param:q") + + budget_threats = [ + t for t in result["threats"] if t["pattern"] == "scan_time_budget_exhausted" + ] + assert budget_threats + assert budget_threats[0]["type"] == "pattern_timeout" + assert budget_threats[0]["weight"] >= config.detection_threat_score_threshold + assert "scan_time_budget_exhausted" in result["timeouts"] + assert result["is_threat"] is True + + +async def test_budget_is_shared_across_fields_of_one_request( + fake_regex_clock: _FakeClock, +) -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig(detection_max_scan_seconds=2.5)) + + with _scan_value_budget(512, max_scan_seconds=2.5): + first = await manager.detect("hello", "127.0.0.1", "query_param:a") + second = await manager.detect("world", "127.0.0.1", "query_param:b") + + names = [t["pattern"] for t in first["threats"] + second["threats"]] + assert "scan_time_budget_exhausted" in names + assert second["is_threat"] is True + + +async def test_fallback_check_honours_budget() -> None: + with _scan_value_budget(512, max_scan_seconds=0.1): + detection_scan._record_scan_time(1.0) + detected, trigger = await detection_scan._fallback_pattern_check( + "hello", "127.0.0.1", "query_param:q" + ) + + assert detected is True + assert "budget" in trigger diff --git a/tests/test_semantic_excluded_headers.py b/tests/test_semantic_excluded_headers.py new file mode 100644 index 00000000..48e7caca --- /dev/null +++ b/tests/test_semantic_excluded_headers.py @@ -0,0 +1,103 @@ +from collections.abc import AsyncGenerator +from typing import Any + +import pytest + +from guard_core.decorators.base import RouteConfig +from guard_core.handlers.suspatterns_handler import SusPatternsManager +from guard_core.models import SecurityConfig +from guard_core.utils import detect_penetration_attempt +from tests.conftest import MockGuardRequest + +_JWT_COOKIE = ( + "session=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" +) + + +@pytest.fixture(autouse=True) +async def _reset_suspatterns_singleton() -> AsyncGenerator[None, None]: + original_instance = SusPatternsManager._instance + original_config = SusPatternsManager._config + + SusPatternsManager._instance = None + SusPatternsManager._config = None + + yield + + SusPatternsManager._instance = original_instance + SusPatternsManager._config = original_config + + +def _config(**kwargs: Any) -> SecurityConfig: + return SecurityConfig(detection_semantic_threshold=0.3, **kwargs) + + +def test_field_defaults_to_empty() -> None: + assert SecurityConfig().excluded_semantic_detection_headers == set() + + +async def test_jwt_cookie_is_flagged_semantically_by_default() -> None: + request = MockGuardRequest(headers={"cookie": _JWT_COOKIE}) + + result = await detect_penetration_attempt(request, _config()) + + assert result.is_threat is True + assert "Semantic attack" in result.trigger_info + + +async def test_listed_header_skips_semantic_analysis_case_insensitively() -> None: + request = MockGuardRequest(headers={"cookie": _JWT_COOKIE}) + + result = await detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={" Cookie "}) + ) + + assert result.is_threat is False + + +async def test_unlisted_header_still_gets_semantic_analysis() -> None: + request = MockGuardRequest(headers={"x-session": _JWT_COOKIE}) + + result = await detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert result.is_threat is True + + +async def test_listed_header_is_still_scanned_by_regex() -> None: + request = MockGuardRequest(headers={"cookie": "id=1' OR '1'='1' UNION SELECT 1--"}) + + result = await detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert result.is_threat is True + assert "Semantic attack" not in result.trigger_info + + +async def test_route_config_set_is_merged() -> None: + request = MockGuardRequest(headers={"cookie": _JWT_COOKIE}) + route_config = RouteConfig() + route_config.excluded_semantic_detection_headers = {"cookie"} + + result = await detect_penetration_attempt(request, _config(), route_config) + + assert result.is_threat is False + + +async def test_exclusion_does_not_leak_to_the_next_request() -> None: + request = MockGuardRequest(headers={"cookie": _JWT_COOKIE}) + config = _config(excluded_semantic_detection_headers={"cookie"}) + + assert (await detect_penetration_attempt(request, config)).is_threat is False + assert (await detect_penetration_attempt(request, _config())).is_threat is True + + +def test_reassignment_rejects_bare_string() -> None: + config = SecurityConfig() + + with pytest.raises(ValueError, match="excluded_semantic_detection_headers"): + config.excluded_semantic_detection_headers = "cookie" # type: ignore[assignment] diff --git a/tests/test_sensitive_data_invariant.py b/tests/test_sensitive_data_invariant.py index f965c38b..20dc8e54 100644 --- a/tests/test_sensitive_data_invariant.py +++ b/tests/test_sensitive_data_invariant.py @@ -1194,6 +1194,23 @@ async def _run_auth_invalid(secret: str, ctx: "_ScenarioContext") -> None: assert response is not None, "auth_invalid scenario never blocked" +async def _run_scan_time_budget_exhausted(secret: str, ctx: "_ScenarioContext") -> None: + config = _scenario_config(detection_max_scan_seconds=0.1) + request = MockGuardRequest( + query_params={"token": f"{secret} {_SQLI}"}, + client_host=_client_ip_for("scan_time_budget_exhausted"), + ) + with patch( + "guard_core._utils.detection_scan._scan_time_remaining", return_value=0.0 + ): + result = await detect_penetration_attempt(request, config) + assert result.is_threat, "scan_time_budget_exhausted scenario never flagged" + assert "scan_time_budget_exhausted" in result.trigger_info + assert any( + "detection_max_scan_seconds" in r.getMessage() for r in ctx.caplog.records + ) + + async def _run_referrer_invalid(secret: str, ctx: "_ScenarioContext") -> None: client_ip = _client_ip_for("referrer_invalid") route_config = RouteConfig() @@ -1798,6 +1815,7 @@ class ComponentScenario: ComponentScenario("auth_missing", _run_auth_missing), ComponentScenario("auth_invalid", _run_auth_invalid), ComponentScenario("referrer_invalid", _run_referrer_invalid), + ComponentScenario("scan_time_budget_exhausted", _run_scan_time_budget_exhausted), ComponentScenario( "referrer_invalid_query_token", _run_referrer_invalid_query_token ), diff --git a/tests/test_sensitive_output_producers.py b/tests/test_sensitive_output_producers.py index 84ef8007..cf8c703d 100644 --- a/tests/test_sensitive_output_producers.py +++ b/tests/test_sensitive_output_producers.py @@ -172,6 +172,9 @@ def group_producer_hits(hits: list[ProducerHit]) -> dict[str, list[ProducerHit]] "size_json_body_300_level_nested_wrapper_must_not_raise", } ), + "_utils/detection_scan.py:_warn_scan_time_cap_reached_once": frozenset( + {"scan_time_budget_exhausted"} + ), "_utils/ip_extraction.py:_warn_forwarded_header_chain_too_short": frozenset( {"xff_spoof_warning"} ), diff --git a/tests/test_sus_patterns/test_pattern_source_redaction.py b/tests/test_sus_patterns/test_pattern_source_redaction.py index 4a680a42..6c9b084b 100644 --- a/tests/test_sus_patterns/test_pattern_source_redaction.py +++ b/tests/test_sus_patterns/test_pattern_source_redaction.py @@ -102,8 +102,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: @@ -130,8 +130,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: @@ -156,8 +156,8 @@ async def test_check_pattern_with_timeout_log_redacts_secret_shaped_pattern( with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: @@ -182,8 +182,8 @@ async def test_check_pattern_with_timeout_exception_log_redacts_secret_shaped_pa with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: diff --git a/tests/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py b/tests/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py index a64f3cf9..5ecaf205 100644 --- a/tests/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py +++ b/tests/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py @@ -3,7 +3,7 @@ import re import time from collections.abc import Iterator -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest @@ -140,8 +140,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: @@ -167,8 +167,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: diff --git a/tests/test_sus_patterns/test_sus_patterns.py b/tests/test_sus_patterns/test_sus_patterns.py index bc5c92e0..64d1359b 100644 --- a/tests/test_sus_patterns/test_sus_patterns.py +++ b/tests/test_sus_patterns/test_sus_patterns.py @@ -343,8 +343,8 @@ async def test_regex_timeout_fallback() -> None: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: @@ -544,23 +544,18 @@ async def test_legacy_pattern_timeout_uses_configured_compiler_timeout( assert manager._compiler is None manager._config = MagicMock(detection_compiler_timeout=7.5) - captured_timeouts = [] + captured_timeouts: list[float] = [] - class FakeFuture: - def result(self, timeout: float = 0) -> None: - captured_timeouts.append(timeout) - raise concurrent.futures.TimeoutError() + async def fake_await(future: object, timeout: float) -> None: + captured_timeouts.append(timeout) + raise concurrent.futures.TimeoutError() - def cancel(self) -> None: - pass - - class FakeExecutor: - def submit(self, fn: object, *args: object) -> "FakeFuture": - return FakeFuture() - - with patch( - "guard_core.handlers._suspatterns_regex.shared_regex_executor", - return_value=FakeExecutor(), + with ( + patch( + "guard_core.handlers._suspatterns_regex._await_regex_future", + side_effect=fake_await, + ), + patch("guard_core.handlers._suspatterns_regex.shared_regex_executor"), ): match, timed_out = await manager._check_pattern_with_timeout( re.compile("x"), "content", "1.2.3.4", 0.0 @@ -587,8 +582,8 @@ async def test_regex_search_exception_fallback() -> None: with patch( "guard_core.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("Test exception") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("Test exception")) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.handlers._suspatterns_regex.logger") as mock_logger: diff --git a/tests/test_sync/test_decorators/test_detection_exclusion.py b/tests/test_sync/test_decorators/test_detection_exclusion.py index b8f2d026..e581685e 100644 --- a/tests/test_sync/test_decorators/test_detection_exclusion.py +++ b/tests/test_sync/test_decorators/test_detection_exclusion.py @@ -100,3 +100,16 @@ def test_detection_exclusion_no_args_creates_route_config() -> None: assert rc.excluded_detection_params is None assert rc.excluded_detection_body_fields is None assert rc.enabled_detection_categories is None + + +def test_detection_exclusion_sets_semantic_headers() -> None: + d = _FakeDecorator(SecurityConfig()) + decorated = _decorate(d.detection_exclusion(semantic_headers={"cookie"})) + rc = d.get_route_config(d._get_route_id(decorated)) + assert rc is not None + assert rc.excluded_semantic_detection_headers == {"cookie"} + assert rc.excluded_detection_headers is None + + +def test_semantic_headers_default_to_none() -> None: + assert RouteConfig().excluded_semantic_detection_headers is None diff --git a/tests/test_sync/test_models/test_every_collection_field_is_revalidated.py b/tests/test_sync/test_models/test_every_collection_field_is_revalidated.py index 35b6bfc9..d37e561f 100644 --- a/tests/test_sync/test_models/test_every_collection_field_is_revalidated.py +++ b/tests/test_sync/test_models/test_every_collection_field_is_revalidated.py @@ -65,6 +65,7 @@ def test_collection_field_is_revalidated_or_immutable(field_name: str) -> None: ("agent_sensitive_headers", ["authorization"], "nope"), ("otel_resource_attributes", {"service.name": "guard"}, "nope"), ("excluded_detection_headers", {"x-foo"}, "nope"), + ("excluded_semantic_detection_headers", {"cookie"}, "nope"), ("excluded_detection_params", {"token"}, "nope"), ("excluded_detection_body_fields", {"password"}, "nope"), ) diff --git a/tests/test_sync/test_scan_time_budget.py b/tests/test_sync/test_scan_time_budget.py new file mode 100644 index 00000000..914ac11b --- /dev/null +++ b/tests/test_sync/test_scan_time_budget.py @@ -0,0 +1,155 @@ +from collections.abc import Generator, Iterator +from typing import Any +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +from guard_core.models import SecurityConfig +from guard_core.sync._utils import detection_scan +from guard_core.sync._utils.detection_scan import ( + _scan_time_budget_exhausted, + _scan_value_budget, +) +from guard_core.sync.handlers.suspatterns_handler import SusPatternsManager +from guard_core.sync.utils import detect_penetration_attempt +from tests.test_sync.conftest import SyncMockGuardRequest + + +@pytest.fixture(autouse=True) +def _reset_suspatterns_singleton() -> Generator[None, None]: + original_instance = SusPatternsManager._instance + original_config = SusPatternsManager._config + + SusPatternsManager._instance = None + SusPatternsManager._config = None + + yield + + SusPatternsManager._instance = original_instance + SusPatternsManager._config = original_config + + +class _FakeClock: + def __init__(self, step: float) -> None: + self.now = 0.0 + self.step = step + + def monotonic(self) -> float: + self.now += self.step + return self.now + + +@pytest.fixture +def fake_regex_clock() -> Iterator[_FakeClock]: + clock = _FakeClock(step=1.0) + with patch("guard_core.sync.handlers._suspatterns_regex.time") as fake_time: + fake_time.monotonic = clock.monotonic + yield clock + + +def test_field_default_and_bounds() -> None: + assert SecurityConfig().detection_max_scan_seconds == 10.0 + assert SecurityConfig(detection_max_scan_seconds=0.1) + assert SecurityConfig(detection_max_scan_seconds=60.0) + for bad in (0.0, 0.09, 60.1): + with pytest.raises(ValidationError): + SecurityConfig(detection_max_scan_seconds=bad) + + +def test_budget_is_unbounded_outside_a_request() -> None: + assert _scan_time_budget_exhausted() is False + + +def test_budget_tracks_recorded_time_and_resets() -> None: + with _scan_value_budget(512, max_scan_seconds=1.0): + assert _scan_time_budget_exhausted() is False + detection_scan._record_scan_time(0.4) + assert _scan_time_budget_exhausted() is False + detection_scan._record_scan_time(0.6) + assert _scan_time_budget_exhausted() is True + assert _scan_time_budget_exhausted() is False + with _scan_value_budget(512, max_scan_seconds=1.0): + assert _scan_time_budget_exhausted() is False + + +def test_default_budget_does_not_flag_benign_request() -> None: + request = SyncMockGuardRequest( + path="/items?page=2", headers={"accept-language": "en-US,en;q=0.9"} + ) + + result = detect_penetration_attempt(request, SecurityConfig()) + + assert result.is_threat is False + + +def test_exhausted_budget_stops_scan_and_flags_request( + fake_regex_clock: _FakeClock, +) -> None: + request = SyncMockGuardRequest( + path="/items", headers={"x-pad": "benign filler value"} + ) + manager = SusPatternsManager() + seen: list[str] = [] + real = manager._check_regex_pattern + + def counting(pattern: Any, *args: Any, **kwargs: Any) -> Any: + seen.append(pattern.pattern) + return real(pattern, *args, **kwargs) + + with patch.object(manager, "_check_regex_pattern", counting): + result = detect_penetration_attempt( + request, SecurityConfig(detection_max_scan_seconds=0.1) + ) + + all_patterns = manager.get_all_compiled_patterns() + assert result.is_threat is True + assert "Pattern exceeded scan time budget" in result.trigger_info + assert len(seen) < len(all_patterns) + assert fake_regex_clock.now > 0 + + +def test_budget_threat_is_weighted_and_reported_as_timeout( + fake_regex_clock: _FakeClock, +) -> None: + config = SecurityConfig(detection_max_scan_seconds=0.1) + manager = SusPatternsManager() + manager.configure(config) + + with _scan_value_budget(512, max_scan_seconds=0.1): + result = manager.detect("hello world", "127.0.0.1", "query_param:q") + + budget_threats = [ + t for t in result["threats"] if t["pattern"] == "scan_time_budget_exhausted" + ] + assert budget_threats + assert budget_threats[0]["type"] == "pattern_timeout" + assert budget_threats[0]["weight"] >= config.detection_threat_score_threshold + assert "scan_time_budget_exhausted" in result["timeouts"] + assert result["is_threat"] is True + + +def test_budget_is_shared_across_fields_of_one_request( + fake_regex_clock: _FakeClock, +) -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig(detection_max_scan_seconds=2.5)) + + with _scan_value_budget(512, max_scan_seconds=2.5): + first = manager.detect("hello", "127.0.0.1", "query_param:a") + second = manager.detect("world", "127.0.0.1", "query_param:b") + + names = [t["pattern"] for t in first["threats"] + second["threats"]] + assert "scan_time_budget_exhausted" in names + assert second["is_threat"] is True + + +def test_fallback_check_honours_budget() -> None: + with _scan_value_budget(512, max_scan_seconds=0.1): + detection_scan._record_scan_time(1.0) + detected, trigger = detection_scan._fallback_pattern_check( + "hello", "127.0.0.1", "query_param:q" + ) + + assert detected is True + assert "budget" in trigger diff --git a/tests/test_sync/test_semantic_excluded_headers.py b/tests/test_sync/test_semantic_excluded_headers.py new file mode 100644 index 00000000..cb6d9cf1 --- /dev/null +++ b/tests/test_sync/test_semantic_excluded_headers.py @@ -0,0 +1,105 @@ +from collections.abc import Generator +from typing import Any + +import pytest + +from guard_core.models import SecurityConfig +from guard_core.sync.decorators.base import RouteConfig +from guard_core.sync.handlers.suspatterns_handler import SusPatternsManager +from guard_core.sync.utils import detect_penetration_attempt +from tests.test_sync.conftest import SyncMockGuardRequest + +_JWT_COOKIE = ( + "session=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" +) + + +@pytest.fixture(autouse=True) +def _reset_suspatterns_singleton() -> Generator[None, None]: + original_instance = SusPatternsManager._instance + original_config = SusPatternsManager._config + + SusPatternsManager._instance = None + SusPatternsManager._config = None + + yield + + SusPatternsManager._instance = original_instance + SusPatternsManager._config = original_config + + +def _config(**kwargs: Any) -> SecurityConfig: + return SecurityConfig(detection_semantic_threshold=0.3, **kwargs) + + +def test_field_defaults_to_empty() -> None: + assert SecurityConfig().excluded_semantic_detection_headers == set() + + +def test_jwt_cookie_is_flagged_semantically_by_default() -> None: + request = SyncMockGuardRequest(headers={"cookie": _JWT_COOKIE}) + + result = detect_penetration_attempt(request, _config()) + + assert result.is_threat is True + assert "Semantic attack" in result.trigger_info + + +def test_listed_header_skips_semantic_analysis_case_insensitively() -> None: + request = SyncMockGuardRequest(headers={"cookie": _JWT_COOKIE}) + + result = detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={" Cookie "}) + ) + + assert result.is_threat is False + + +def test_unlisted_header_still_gets_semantic_analysis() -> None: + request = SyncMockGuardRequest(headers={"x-session": _JWT_COOKIE}) + + result = detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert result.is_threat is True + + +def test_listed_header_is_still_scanned_by_regex() -> None: + request = SyncMockGuardRequest( + headers={"cookie": "id=1' OR '1'='1' UNION SELECT 1--"} + ) + + result = detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert result.is_threat is True + assert "Semantic attack" not in result.trigger_info + + +def test_route_config_set_is_merged() -> None: + request = SyncMockGuardRequest(headers={"cookie": _JWT_COOKIE}) + route_config = RouteConfig() + route_config.excluded_semantic_detection_headers = {"cookie"} + + result = detect_penetration_attempt(request, _config(), route_config) + + assert result.is_threat is False + + +def test_exclusion_does_not_leak_to_the_next_request() -> None: + request = SyncMockGuardRequest(headers={"cookie": _JWT_COOKIE}) + config = _config(excluded_semantic_detection_headers={"cookie"}) + + assert (detect_penetration_attempt(request, config)).is_threat is False + assert (detect_penetration_attempt(request, _config())).is_threat is True + + +def test_reassignment_rejects_bare_string() -> None: + config = SecurityConfig() + + with pytest.raises(ValueError, match="excluded_semantic_detection_headers"): + config.excluded_semantic_detection_headers = "cookie" # type: ignore[assignment] diff --git a/tests/test_sync/test_sensitive_data_invariant.py b/tests/test_sync/test_sensitive_data_invariant.py index 68983409..b9f7341a 100644 --- a/tests/test_sync/test_sensitive_data_invariant.py +++ b/tests/test_sync/test_sensitive_data_invariant.py @@ -1198,6 +1198,23 @@ def _run_auth_invalid(secret: str, ctx: "_ScenarioContext") -> None: assert response is not None, "auth_invalid scenario never blocked" +def _run_scan_time_budget_exhausted(secret: str, ctx: "_ScenarioContext") -> None: + config = _scenario_config(detection_max_scan_seconds=0.1) + request = SyncMockGuardRequest( + query_params={"token": f"{secret} {_SQLI}"}, + client_host=_client_ip_for("scan_time_budget_exhausted"), + ) + with patch( + "guard_core.sync._utils.detection_scan._scan_time_remaining", return_value=0.0 + ): + result = detect_penetration_attempt(request, config) + assert result.is_threat, "scan_time_budget_exhausted scenario never flagged" + assert "scan_time_budget_exhausted" in result.trigger_info + assert any( + "detection_max_scan_seconds" in r.getMessage() for r in ctx.caplog.records + ) + + def _run_referrer_invalid(secret: str, ctx: "_ScenarioContext") -> None: client_ip = _client_ip_for("referrer_invalid") route_config = RouteConfig() @@ -1790,6 +1807,7 @@ class ComponentScenario: ComponentScenario("auth_missing", _run_auth_missing), ComponentScenario("auth_invalid", _run_auth_invalid), ComponentScenario("referrer_invalid", _run_referrer_invalid), + ComponentScenario("scan_time_budget_exhausted", _run_scan_time_budget_exhausted), ComponentScenario( "referrer_invalid_query_token", _run_referrer_invalid_query_token ), diff --git a/tests/test_sync/test_sensitive_output_producers.py b/tests/test_sync/test_sensitive_output_producers.py index 84ef8007..cf8c703d 100644 --- a/tests/test_sync/test_sensitive_output_producers.py +++ b/tests/test_sync/test_sensitive_output_producers.py @@ -172,6 +172,9 @@ def group_producer_hits(hits: list[ProducerHit]) -> dict[str, list[ProducerHit]] "size_json_body_300_level_nested_wrapper_must_not_raise", } ), + "_utils/detection_scan.py:_warn_scan_time_cap_reached_once": frozenset( + {"scan_time_budget_exhausted"} + ), "_utils/ip_extraction.py:_warn_forwarded_header_chain_too_short": frozenset( {"xff_spoof_warning"} ), diff --git a/tests/test_sync/test_sus_patterns/test_pattern_source_redaction.py b/tests/test_sync/test_sus_patterns/test_pattern_source_redaction.py index 1df76267..d1237abd 100644 --- a/tests/test_sync/test_sus_patterns/test_pattern_source_redaction.py +++ b/tests/test_sync/test_sus_patterns/test_pattern_source_redaction.py @@ -103,8 +103,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: @@ -131,8 +131,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: @@ -157,8 +157,8 @@ def test_check_pattern_with_timeout_log_redacts_secret_shaped_pattern( with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: @@ -183,8 +183,8 @@ def test_check_pattern_with_timeout_exception_log_redacts_secret_shaped_pattern( with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: diff --git a/tests/test_sync/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py b/tests/test_sync/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py index 78947e9d..d6fb0b7c 100644 --- a/tests/test_sync/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py +++ b/tests/test_sync/test_sus_patterns/test_shell_dash_c_var_assignment_newline_prefix.py @@ -3,7 +3,7 @@ import re import time from collections.abc import Iterator -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest @@ -135,8 +135,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = concurrent.futures.TimeoutError() + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(concurrent.futures.TimeoutError()) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: @@ -161,8 +161,8 @@ def _finder(_content: str) -> Iterator[re.Match[str]]: with patch( "guard_core.sync.handlers._suspatterns_regex.shared_regex_executor" ) as mock_shared_executor: - mock_future = MagicMock() - mock_future.result.side_effect = RuntimeError("boom") + mock_future: concurrent.futures.Future[None] = concurrent.futures.Future() + mock_future.set_exception(RuntimeError("boom")) mock_shared_executor.return_value.submit.return_value = mock_future with patch("guard_core.sync.handlers._suspatterns_regex.logger") as mock_logger: From 980105146f98417501b1c9f9f395b8503ee4f39a Mon Sep 17 00:00:00 2001 From: Renn F Date: Thu, 15 Oct 2026 03:05:49 +0200 Subject: [PATCH 2/5] fix(detection): name the scan time budget in the trigger when it tips a request When a per-pattern timeout or the spent scan budget was what pushed a request over the threshold, the trigger named only the first threat, so the block read as an ordinary pattern match. The trigger now appends the first pattern_timeout threat behind a non-timeout first threat. The sync tree is regenerated with scripts/unasync.py. --- guard_core/_utils/detection_scan.py | 12 ++++++++- guard_core/sync/_utils/detection_scan.py | 12 ++++++++- tests/test_scan_time_budget.py | 32 ++++++++++++++++++++++++ tests/test_sync/test_scan_time_budget.py | 32 ++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/guard_core/_utils/detection_scan.py b/guard_core/_utils/detection_scan.py index f71e05cd..71f9a3d4 100644 --- a/guard_core/_utils/detection_scan.py +++ b/guard_core/_utils/detection_scan.py @@ -277,6 +277,16 @@ def _build_threat_message(threat: dict[str, Any]) -> str: return "Threat detected" +def _build_threats_message(threats: list[dict[str, Any]]) -> str: + message = _build_threat_message(threats[0]) + if threats[0]["type"] == "pattern_timeout": + return message + timeout = next((t for t in threats[1:] if t["type"] == "pattern_timeout"), None) + if timeout is None: + return message + return f"{message}; {_build_threat_message(timeout)}" + + _MAX_USER_AGENT_MATCH_LENGTH = 512 @@ -421,7 +431,7 @@ async def _check_value_enhanced( threats: list[dict] = list(result.get("threats", [])) if threats: - return True, _build_threat_message(threats[0]), threats, None + return True, _build_threats_message(threats), threats, None return True, "Threat detected", threats, None diff --git a/guard_core/sync/_utils/detection_scan.py b/guard_core/sync/_utils/detection_scan.py index 28d213f5..8d4d433c 100644 --- a/guard_core/sync/_utils/detection_scan.py +++ b/guard_core/sync/_utils/detection_scan.py @@ -277,6 +277,16 @@ def _build_threat_message(threat: dict[str, Any]) -> str: return "Threat detected" +def _build_threats_message(threats: list[dict[str, Any]]) -> str: + message = _build_threat_message(threats[0]) + if threats[0]["type"] == "pattern_timeout": + return message + timeout = next((t for t in threats[1:] if t["type"] == "pattern_timeout"), None) + if timeout is None: + return message + return f"{message}; {_build_threat_message(timeout)}" + + _MAX_USER_AGENT_MATCH_LENGTH = 512 @@ -419,7 +429,7 @@ def _check_value_enhanced( threats: list[dict] = list(result.get("threats", [])) if threats: - return True, _build_threat_message(threats[0]), threats, None + return True, _build_threats_message(threats), threats, None return True, "Threat detected", threats, None diff --git a/tests/test_scan_time_budget.py b/tests/test_scan_time_budget.py index fea95623..26c33b1f 100644 --- a/tests/test_scan_time_budget.py +++ b/tests/test_scan_time_budget.py @@ -151,3 +151,35 @@ async def test_fallback_check_honours_budget() -> None: assert detected is True assert "budget" in trigger + + +async def test_trigger_names_the_budget_when_it_tips_the_request( + fake_regex_clock: _FakeClock, +) -> None: + request = MockGuardRequest(path="/items", headers={"x-pad": "select a from b"}) + + result = await detect_penetration_attempt( + request, SecurityConfig(detection_max_scan_seconds=0.1) + ) + + assert result.is_threat is True + assert "Pattern exceeded scan time budget" in result.trigger_info + assert "scan_time_budget_exhausted" in result.trigger_info + + +def test_threats_message_appends_the_first_later_timeout() -> None: + regex = {"type": "regex", "pattern": "a"} + timeout = {"type": "pattern_timeout", "pattern": "slow"} + other = {"type": "pattern_timeout", "pattern": "later"} + + message = detection_scan._build_threats_message([regex, timeout, other]) + + assert message == ( + "Value matched pattern 'a'; Pattern exceeded scan time budget: 'slow'" + ) + assert detection_scan._build_threats_message([regex]) == ( + "Value matched pattern 'a'" + ) + assert detection_scan._build_threats_message([timeout, regex]) == ( + "Pattern exceeded scan time budget: 'slow'" + ) diff --git a/tests/test_sync/test_scan_time_budget.py b/tests/test_sync/test_scan_time_budget.py index 914ac11b..84f65c6d 100644 --- a/tests/test_sync/test_scan_time_budget.py +++ b/tests/test_sync/test_scan_time_budget.py @@ -153,3 +153,35 @@ def test_fallback_check_honours_budget() -> None: assert detected is True assert "budget" in trigger + + +def test_trigger_names_the_budget_when_it_tips_the_request( + fake_regex_clock: _FakeClock, +) -> None: + request = SyncMockGuardRequest(path="/items", headers={"x-pad": "select a from b"}) + + result = detect_penetration_attempt( + request, SecurityConfig(detection_max_scan_seconds=0.1) + ) + + assert result.is_threat is True + assert "Pattern exceeded scan time budget" in result.trigger_info + assert "scan_time_budget_exhausted" in result.trigger_info + + +def test_threats_message_appends_the_first_later_timeout() -> None: + regex = {"type": "regex", "pattern": "a"} + timeout = {"type": "pattern_timeout", "pattern": "slow"} + other = {"type": "pattern_timeout", "pattern": "later"} + + message = detection_scan._build_threats_message([regex, timeout, other]) + + assert message == ( + "Value matched pattern 'a'; Pattern exceeded scan time budget: 'slow'" + ) + assert detection_scan._build_threats_message([regex]) == ( + "Value matched pattern 'a'" + ) + assert detection_scan._build_threats_message([timeout, regex]) == ( + "Pattern exceeded scan time budget: 'slow'" + ) From 43726f5a49c030aa295d8177d368896d0558ab12 Mon Sep 17 00:00:00 2001 From: Renn F Date: Thu, 15 Oct 2026 04:07:25 +0200 Subject: [PATCH 3/5] fix(detection): budget the scan on CPU time, not wall time detection_max_scan_seconds summed the wall time of each pattern, so on a busy or CPU-starved host ordinary requests exceeded the default and failed closed (600 benign query params: 26.7 s wall for 3.7 s of CPU). The budget now accumulates thread CPU time: inline scans measure time.thread_time() deltas, and executor-run scans measure it inside the worker and return it, since contextvars do not cross into the executor. A scan that hits detection_compiler_timeout is charged that timeout. The per-pattern timeout is no longer clamped to the budget left (the units no longer match), and event loop slicing stays wall based. The sync tree is regenerated with scripts/unasync.py. --- docs/configuration/detection-tuning.md | 2 +- docs/configuration/security-config.md | 2 +- docs/internals/detection-engine.md | 2 +- .../skills/guard-core/references/config.md | 2 +- .../skills/guard-core/references/detection.md | 2 +- guard_core/_security_config_fields.py | 4 +- guard_core/_utils/detection_scan.py | 51 ++++++++++++++++--- guard_core/detection_engine/compiler.py | 11 ++-- guard_core/handlers/_suspatterns_regex.py | 42 ++++++++------- guard_core/sync/_utils/detection_scan.py | 51 ++++++++++++++++--- guard_core/sync/detection_engine/compiler.py | 11 ++-- .../sync/handlers/_suspatterns_regex.py | 42 ++++++++------- tests/test_scan_time_budget.py | 40 +++++++++++++-- tests/test_shared_regex_executor.py | 10 +++- .../test_custom_redos_hardening.py | 19 ++++++- tests/test_sync/test_scan_time_budget.py | 40 +++++++++++++-- tests/test_sync/test_shared_regex_executor.py | 10 +++- .../test_custom_redos_hardening.py | 19 ++++++- 18 files changed, 285 insertions(+), 75 deletions(-) diff --git a/docs/configuration/detection-tuning.md b/docs/configuration/detection-tuning.md index ac246b15..2c358305 100644 --- a/docs/configuration/detection-tuning.md +++ b/docs/configuration/detection-tuning.md @@ -29,7 +29,7 @@ Maximum time in seconds for a single regex pattern match. Patterns that exceed t **Type**: `float` | **Default**: `10.0` | **Range**: 0.1 - 60.0 -Total regex scan time allowed for one request, summed across every field (query parameters, headers, body values) and every pattern, not per `detect()` call. Once the budget is spent, the remaining patterns and request values are not scanned and the request gets a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the reported `timeouts`, the same way a per-pattern `detection_compiler_timeout` does, so padding a request with filler cannot silently skip detection. The wait for a single pattern is also clamped to the time left in the budget. The default is far above normal traffic (a few milliseconds per request, roughly 16 ms for 8,863 characters on one measured CPU) and about 6 times the worst legitimate case the value cap allows (256 benign query parameters, 512 scan units, measured at about 1.5 s on that CPU), which leaves headroom for slower hosts. Only regex pattern time counts, not decoding or semantic analysis. +CPU time allowed for regex scanning one request (thread CPU time, not wall time, so a busy or CPU-starved host does not exhaust it), summed across every field (query parameters, headers, body values) and every pattern, not per `detect()` call. Once the budget is spent, the remaining patterns and request values are not scanned and the request gets a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the reported `timeouts`, the same way a per-pattern `detection_compiler_timeout` does, so padding a request with filler cannot silently skip detection. The default is far above normal traffic (a few milliseconds per request, roughly 16 ms for 8,863 characters on one measured CPU) and about 6 times the worst legitimate case the value cap allows (256 benign query parameters, 512 scan units, measured at about 1.5 s on that CPU), which leaves headroom for slower hosts. Only regex pattern CPU time counts, not decoding or semantic analysis. A pattern that hits `detection_compiler_timeout` is charged that timeout. In the async tree the scan awaits the regex worker instead of blocking the event loop, so other requests run between patterns. A single regex that is already executing still holds the GIL until it returns. The sync tree has no event loop and keeps blocking. diff --git a/docs/configuration/security-config.md b/docs/configuration/security-config.md index ab9157c2..d7aea7ed 100644 --- a/docs/configuration/security-config.md +++ b/docs/configuration/security-config.md @@ -394,7 +394,7 @@ Detection Engine | `detection_max_tracked_patterns` | `int` | `1000` | 100 - 5000 | Maximum patterns to track for performance. | | `detection_max_body_inspect_bytes` | `int` | `262144`| 1024 - 10485760 | Body size cap read/scanned for detection; distinct from `detection_max_content_length` and `max_request_size`. A body whose declared `Content-Length` exceeds this cap is read through the adapter's bounded reader (`read_body_prefix`) and only the first this-many bytes are scanned, if the adapter implements one; without a bounded reader the body is not read at all, the same skip as before this cap existed. Either way a one-time warning names the cap, the client, and which of the two happened. A signature that straddles the cap boundary is still caught: a small overlap (the longest compiled pattern length, up to 256 bytes) is read past the cap on every path, including a declared oversized `Content-Length`. | | `detection_max_scan_values` | `int` | `512` | 2 - 100000 | Maximum values (query params, headers, JSON keys/values, form/multipart fields) scanned per request; remaining values are skipped and a one-time warning logs the client IP once reached. Each named value costs two scan units (name, then value), so the minimum is 2. | -| `detection_max_scan_seconds` | `float` | `10.0` | 0.1 - 60.0 | Cap on the total regex scan time spent on one request, summed across every field and pattern (not per `detect()` call). Once reached, remaining patterns and values are not scanned and the request receives a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the event `timeouts`, so padding a request cannot skip detection; a one-time warning logs the client IP. A single pattern's wait is also clamped to the time left. Only regex pattern time is counted. | +| `detection_max_scan_seconds` | `float` | `10.0` | 0.1 - 60.0 | Cap on the CPU time spent regex scanning one request (not wall time, so a busy or CPU-starved host does not exhaust it), summed across every field and pattern (not per `detect()` call). Once reached, remaining patterns and values are not scanned and the request receives a weighted `pattern_timeout` threat (pattern `scan_time_budget_exhausted`) that counts toward the anomaly score and the event `timeouts`, so padding a request cannot skip detection; a one-time warning logs the client IP. Only regex pattern CPU time is counted. | | `detection_max_scan_chars` | `int` | `65536` | 1024 - 262144 | Maximum total characters, across every value handed to the pattern engine per request (counted at the same point as `detection_max_scan_values`), before remaining values are skipped and a one-time warning logs the client IP. A value already in progress when the budget is checked is always scanned in full; only values that would start after the budget is spent are skipped, so a single large value is never silently dropped (GHSA-3hfx-8m47-5f9h residual). | | `detection_max_json_depth` | `int` | `32` | 1 - 1000 | Maximum nesting depth of a JSON request body walked structurally. A dict or list reached at this depth is not descended into: it is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, so content hidden below the cap is still scanned as text rather than structurally. A one-time warning logs the client IP once reached (GHSA-f6cf-jjhc-qp85). | | `detection_binary_min_run_length` | `int` | `16` | 4 - 1024 | Minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned. A part payload whose binary artifact characters make up at least a fifth of it is reduced to its printable runs before scanning; compressed bytes produce attack-shaped printable fragments at a rate that grows with file size, so shorter runs are skipped instead of matched. Text uploads and short or mostly-text payloads stay below the ratio and keep their full scan, whole-body fallback scans are never reduced (raw-body signature coverage stays intact), and text embedded in uploads (scripts inside PDFs, stored paths in archives) forms runs past the threshold and is scanned in full. File names, multipart field names, and part headers are scanned regardless (see `docs/configuration/detection-tuning.md`). | diff --git a/docs/internals/detection-engine.md b/docs/internals/detection-engine.md index 9cbeae14..86746273 100644 --- a/docs/internals/detection-engine.md +++ b/docs/internals/detection-engine.md @@ -48,7 +48,7 @@ flowchart TD Built-in (compile-time-vetted) patterns match directly via `pattern.search()`, with no per-match thread or timeout. Only custom patterns (added via `add_pattern(..., custom=True)`) are routed through the safe-matcher timeout wrapper, a custom-pattern match is flagged as timed out once elapsed time reaches `0.9 * compiler.default_timeout` (i.e. `0.9 * detection_compiler_timeout` when a `SecurityConfig` is wired; legacy/no-compiler mode falls back to a hardcoded 2.0s field default). -The async tree awaits each shared-executor future (`_await_regex_future` in `guard_core/detection_engine/compiler.py`) rather than calling `future.result`, so the event loop runs other tasks between patterns; the sync mirror keeps the blocking `future.result` (the unasync script swaps the helper). A per-request regex time budget (`detection_max_scan_seconds`, tracked in request-scoped context variables by `_scan_value_budget`) stops the pattern loop once spent and emits a weighted `pattern_timeout` threat named `scan_time_budget_exhausted`; `excluded_semantic_detection_headers` skips only the semantic analyzer for the named headers. +The async tree awaits each shared-executor future (`_await_regex_future` in `guard_core/detection_engine/compiler.py`) rather than calling `future.result`, so the event loop runs other tasks between patterns; the sync mirror keeps the blocking `future.result` (the unasync script swaps the helper). A per-request regex CPU time budget (`detection_max_scan_seconds`, tracked in request-scoped context variables by `_scan_value_budget`, fed by thread CPU time measured around inline scans and inside the executor worker) stops the pattern loop once spent and emits a weighted `pattern_timeout` threat named `scan_time_budget_exhausted`; `excluded_semantic_detection_headers` skips only the semantic analyzer for the named headers. When no `SecurityConfig` is wired at construction (legacy mode), `SusPatternsManager` builds neither a `PatternCompiler` nor a `ContentPreprocessor`; content is instead truncated to `detection_max_content_length` (default 10000) via direct slicing before matching, and every pattern, including what would otherwise be a "custom" pattern, runs through the same shared-executor timeout path described above (`_check_pattern_with_timeout`), reporting to the same success/timeout counters. Running detection unconfigured this way is deprecated and will be removed in a future major release: the first time `detect()` actually executes in this state, it raises a `DeprecationWarning` and logs an identical `logger.warning` naming the reason (`_LEGACY_DETECTION_WARNING`, checked by the `_warn_if_legacy_detection` helper in `guard_core/handlers/suspatterns_handler.py`), once per process, a module-level flag shared by every `SusPatternsManager` instance, not a per-instance one. The check runs from inside `detect()` itself, not at construction or at `configure()`, since the module-level singleton is always built unconfigured at import time and only gets a `SecurityConfig` later from the pipeline; a normally initialised pipeline configures the singleton (directly via `sus_patterns_handler.configure(config)`, or implicitly through `detect_penetration_attempt(request, config)`) before it ever runs detection, so it never warns. diff --git a/guard_core/.agents/skills/guard-core/references/config.md b/guard_core/.agents/skills/guard-core/references/config.md index b8f282e3..fb636db7 100644 --- a/guard_core/.agents/skills/guard-core/references/config.md +++ b/guard_core/.agents/skills/guard-core/references/config.md @@ -39,7 +39,7 @@ In-memory rate-limit stores (`RateLimitManager.request_timestamps` and the modul ### Detection -`detection_compiler_timeout` (default `2.0`, range `0.1`-`10.0`; per-pattern match timeout), `detection_max_scan_seconds` (default `10.0`, range `0.1`-`60.0`; total regex scan time per request across all fields and patterns; once spent, remaining patterns and values are not scanned and a weighted `pattern_timeout` threat named `scan_time_budget_exhausted` is added, counted in the anomaly score and `timeouts` -- `guard_core/_utils/detection_scan.py`), `detection_pattern_validation_cache_path` (default `None`; opt-in file path for a disk-backed pattern-validation cache: the empirical cost-verdict outcome of `validate_pattern_safety` is cached keyed by pattern, flags, and engine version so boots reuse prior certifications instead of re-timing every custom pattern, while the deterministic layers always re-run; entries from a different engine version are ignored and overwritten -- `guard_core/detection_engine/_validation_cache.py`), `detection_max_content_length` (default `10000`), `detection_max_body_inspect_bytes` (default `262144`; a body whose declared `Content-Length` exceeds this is read through the adapter's bounded reader, `read_body_prefix`, and only the first this-many bytes are scanned, if the adapter implements one -- without a bounded reader the body is not read at all, the same skip as before this cap existed -- with a one-time `logger.warning` naming the cap, the client, and which of the two happened), `detection_max_scan_values` (default `512`, `ge=2`, `le=100_000`; caps the total query-parameter, header, JSON-key/value, form-field, and multipart-part values scanned per request, including JSON embedded in a single value; each named value costs two scan units, name and value, so `2` is the floor; once reached, remaining request values are not scanned and a one-time `logger.warning` names the client IP -- `guard_core/_security_config_fields.py`, `guard_core/_utils/detection_scan.py`), `detection_max_scan_chars` (default `65536`, `ge=1024`, `le=262144`; caps the total characters across every value handed to the pattern engine per request, at the same accounting point as `detection_max_scan_values`; a value already in progress when checked is always scanned in full, only values that would start after the budget is spent are skipped, so a single large value is never silently dropped -- GHSA-3hfx-8m47-5f9h residual), `detection_max_json_depth` (default `32`, `ge=1`, `le=1000`; caps how deep the structural JSON-body walk descends, an explicit-stack iteration with no recursion; a dict or list reached at this depth is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, and a one-time `logger.warning` names the client IP -- GHSA-f6cf-jjhc-qp85, CWE-674, `guard_core/_security_config_fields.py`, `guard_core/_utils/body_content_scan.py`), `detection_binary_min_run_length` (default `16`, `ge=4`, `le=1024`; minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned: a part payload whose binary artifact characters fill at least a fifth of it is reduced to its printable runs, while text uploads, short or mostly-text payloads, text parts without a filename, and whole-body fallback scans keep their full scan so raw-body signature coverage (pickle opcodes, UTF-16/32 wide-encoding payloads, null-byte LDAP shapes) stays intact; text embedded in a binary-dense upload forms runs past the threshold and is scanned in full, while attack-shaped fragments shorter than the run inside compressed data are skipped as statistical noise -- `guard_core/detection_engine/binary_islands.py`, `guard_core/_utils/body_form_scan.py`, `guard_core/_utils/detection_scan.py`), `detection_preserve_attack_patterns` (default `True`), `detection_semantic_threshold` (default `0.7`), `detection_anomaly_threshold` (default `3.0`), `detection_slow_pattern_threshold` (default `0.1`), `detection_monitor_history_size` (default `1000`), `detection_max_tracked_patterns` (default `1000`), `detection_threat_score_threshold` (default `1.0`), `detection_scan_body` (default `True`), `enabled_detection_categories` (`frozenset[str]`, subset of the 19 categories; defaults to all 19), `enable_penetration_detection` (default `True`), `enable_dynamic_rules` (default `False`), `dynamic_rules_cache_path` (default `None`; `Path | None`, opt-in file path for the last-known dynamic rules snapshot that is persisted on every successful apply and hydrated once at `initialize_agent`, Redis first then the file; see the SecurityConfig section of SKILL.md; the dump keeps only the fields guard-core's own `DynamicRules` mirror models, so guard-agent-only fields on the live rules object, `emergency_whitelist_only`, `message`, never break persistence), `body_read_timeout` (default `3.0`, `gt=0.0`, `le=30.0`; seconds an adapter's `read_body_prefix`/`body` call is given before the body is treated as unavailable -- bounds the async tree via `asyncio.wait_for`, and in the sync tree bounds how long the caller joins the daemon thread the read runs on), `sync_body_read_max_concurrent` (default `64`, `ge=1`, `le=10000`; cap on daemon threads the sync tree may have blocked at once inside an adapter body read). See `docs/configuration/detection-tuning.md` in the guard-core repository for the full field-by-field tuning guide. +`detection_compiler_timeout` (default `2.0`, range `0.1`-`10.0`; per-pattern match timeout), `detection_max_scan_seconds` (default `10.0`, range `0.1`-`60.0`; total regex scan CPU time per request across all fields and patterns; once spent, remaining patterns and values are not scanned and a weighted `pattern_timeout` threat named `scan_time_budget_exhausted` is added, counted in the anomaly score and `timeouts` -- `guard_core/_utils/detection_scan.py`), `detection_pattern_validation_cache_path` (default `None`; opt-in file path for a disk-backed pattern-validation cache: the empirical cost-verdict outcome of `validate_pattern_safety` is cached keyed by pattern, flags, and engine version so boots reuse prior certifications instead of re-timing every custom pattern, while the deterministic layers always re-run; entries from a different engine version are ignored and overwritten -- `guard_core/detection_engine/_validation_cache.py`), `detection_max_content_length` (default `10000`), `detection_max_body_inspect_bytes` (default `262144`; a body whose declared `Content-Length` exceeds this is read through the adapter's bounded reader, `read_body_prefix`, and only the first this-many bytes are scanned, if the adapter implements one -- without a bounded reader the body is not read at all, the same skip as before this cap existed -- with a one-time `logger.warning` naming the cap, the client, and which of the two happened), `detection_max_scan_values` (default `512`, `ge=2`, `le=100_000`; caps the total query-parameter, header, JSON-key/value, form-field, and multipart-part values scanned per request, including JSON embedded in a single value; each named value costs two scan units, name and value, so `2` is the floor; once reached, remaining request values are not scanned and a one-time `logger.warning` names the client IP -- `guard_core/_security_config_fields.py`, `guard_core/_utils/detection_scan.py`), `detection_max_scan_chars` (default `65536`, `ge=1024`, `le=262144`; caps the total characters across every value handed to the pattern engine per request, at the same accounting point as `detection_max_scan_values`; a value already in progress when checked is always scanned in full, only values that would start after the budget is spent are skipped, so a single large value is never silently dropped -- GHSA-3hfx-8m47-5f9h residual), `detection_max_json_depth` (default `32`, `ge=1`, `le=1000`; caps how deep the structural JSON-body walk descends, an explicit-stack iteration with no recursion; a dict or list reached at this depth is serialized back to text and scanned as one value instead, bounded by `detection_max_content_length`, and a one-time `logger.warning` names the client IP -- GHSA-f6cf-jjhc-qp85, CWE-674, `guard_core/_security_config_fields.py`, `guard_core/_utils/body_content_scan.py`), `detection_binary_min_run_length` (default `16`, `ge=4`, `le=1024`; minimum length a printable run inside a binary-dense multipart file-part payload must reach to be pattern-scanned: a part payload whose binary artifact characters fill at least a fifth of it is reduced to its printable runs, while text uploads, short or mostly-text payloads, text parts without a filename, and whole-body fallback scans keep their full scan so raw-body signature coverage (pickle opcodes, UTF-16/32 wide-encoding payloads, null-byte LDAP shapes) stays intact; text embedded in a binary-dense upload forms runs past the threshold and is scanned in full, while attack-shaped fragments shorter than the run inside compressed data are skipped as statistical noise -- `guard_core/detection_engine/binary_islands.py`, `guard_core/_utils/body_form_scan.py`, `guard_core/_utils/detection_scan.py`), `detection_preserve_attack_patterns` (default `True`), `detection_semantic_threshold` (default `0.7`), `detection_anomaly_threshold` (default `3.0`), `detection_slow_pattern_threshold` (default `0.1`), `detection_monitor_history_size` (default `1000`), `detection_max_tracked_patterns` (default `1000`), `detection_threat_score_threshold` (default `1.0`), `detection_scan_body` (default `True`), `enabled_detection_categories` (`frozenset[str]`, subset of the 19 categories; defaults to all 19), `enable_penetration_detection` (default `True`), `enable_dynamic_rules` (default `False`), `dynamic_rules_cache_path` (default `None`; `Path | None`, opt-in file path for the last-known dynamic rules snapshot that is persisted on every successful apply and hydrated once at `initialize_agent`, Redis first then the file; see the SecurityConfig section of SKILL.md; the dump keeps only the fields guard-core's own `DynamicRules` mirror models, so guard-agent-only fields on the live rules object, `emergency_whitelist_only`, `message`, never break persistence), `body_read_timeout` (default `3.0`, `gt=0.0`, `le=30.0`; seconds an adapter's `read_body_prefix`/`body` call is given before the body is treated as unavailable -- bounds the async tree via `asyncio.wait_for`, and in the sync tree bounds how long the caller joins the daemon thread the read runs on), `sync_body_read_max_concurrent` (default `64`, `ge=1`, `le=10000`; cap on daemon threads the sync tree may have blocked at once inside an adapter body read). See `docs/configuration/detection-tuning.md` in the guard-core repository for the full field-by-field tuning guide. `PatternCompiler.validate_pattern_safety`'s own probe/wait budget (50ms cost-arbiter soft budget, 2.0s killable-subprocess hard timeout) is a hardcoded constant, not a `SecurityConfig` field; see [the detection reference](detection.md#redos-safety) for the mechanism. diff --git a/guard_core/.agents/skills/guard-core/references/detection.md b/guard_core/.agents/skills/guard-core/references/detection.md index 30321fc3..82d5aece 100644 --- a/guard_core/.agents/skills/guard-core/references/detection.md +++ b/guard_core/.agents/skills/guard-core/references/detection.md @@ -100,7 +100,7 @@ This validation subprocess mechanism is unrelated to, and isolated from, the liv ## Tuning knobs (SecurityConfig) -`detection_max_content_length` (default 10000), `detection_compiler_timeout` (default 2.0s, range 0.1-10.0), `detection_max_scan_seconds` (default 10.0s, range 0.1-60.0; per-request total regex time budget, exhaustion adds a weighted `pattern_timeout` threat `scan_time_budget_exhausted`), `excluded_semantic_detection_headers` (opt-in; header values skip the semantic analyzer only), `detection_threat_score_threshold` (default 1.0), `detection_max_scan_values` (default 512, `ge=2`; see [Request-value scan cap](#request-value-scan-cap) above), `enabled_detection_categories` (`frozenset[str]`, subset of the 19, defaults to all 19). `PatternCompiler.validate_pattern_safety`'s 50ms cost-arbiter budget is a hardcoded constant, not a config field; only `max_content_length` (the caller's real content-length cap) is passed in per call. +`detection_max_content_length` (default 10000), `detection_compiler_timeout` (default 2.0s, range 0.1-10.0), `detection_max_scan_seconds` (default 10.0s, range 0.1-60.0; per-request total regex CPU time budget, exhaustion adds a weighted `pattern_timeout` threat `scan_time_budget_exhausted`), `excluded_semantic_detection_headers` (opt-in; header values skip the semantic analyzer only), `detection_threat_score_threshold` (default 1.0), `detection_max_scan_values` (default 512, `ge=2`; see [Request-value scan cap](#request-value-scan-cap) above), `enabled_detection_categories` (`frozenset[str]`, subset of the 19, defaults to all 19). `PatternCompiler.validate_pattern_safety`'s 50ms cost-arbiter budget is a hardcoded constant, not a config field; only `max_content_length` (the caller's real content-length cap) is passed in per call. ## Adding patterns diff --git a/guard_core/_security_config_fields.py b/guard_core/_security_config_fields.py index 0539c8ff..f9448661 100644 --- a/guard_core/_security_config_fields.py +++ b/guard_core/_security_config_fields.py @@ -761,8 +761,8 @@ class _SecurityConfigFields(BaseModel): detection_max_scan_seconds: float = Field( default=10.0, description=( - "Cap on the total regex scan time (seconds) spent on one request, " - "summed across every field and every pattern. Once reached, " + "Cap on the CPU time (seconds) spent regex scanning one request, " + "not wall time, summed across every field and every pattern. Once reached, " "remaining patterns and values are not scanned and the request " "receives a weighted pattern_timeout threat that counts toward the " "anomaly score, so padding a request cannot skip detection." diff --git a/guard_core/_utils/detection_scan.py b/guard_core/_utils/detection_scan.py index 71f9a3d4..148cda27 100644 --- a/guard_core/_utils/detection_scan.py +++ b/guard_core/_utils/detection_scan.py @@ -2,7 +2,7 @@ import logging import re import time -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any @@ -105,7 +105,6 @@ def _scan_time_remaining() -> float | None: _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN = "scan_time_budget_exhausted" -_MIN_CLAMPED_TIMEOUT = 0.001 _EVENT_LOOP_SLICE_SECONDS = 0.002 @@ -121,17 +120,36 @@ def _scan_time_budget_exhausted_threat(weight: float) -> dict[str, Any]: } -def _clamp_to_scan_time_budget(timeout: float) -> float: - remaining = _scan_time_remaining() - if remaining is None: - return timeout - return max(min(timeout, remaining), _MIN_CLAMPED_TIMEOUT) +def _thread_cpu_seconds() -> float: + return time.thread_time() def _record_scan_time(elapsed: float) -> None: _scan_time_spent.set(_scan_time_spent.get() + elapsed) +@contextmanager +def _record_inline_cpu() -> Iterator[None]: + start = _thread_cpu_seconds() + try: + yield + finally: + _record_scan_time(_thread_cpu_seconds() - start) + + +def _timed_in_worker( + work: Callable[[], Any], cpu_sink: list[float] +) -> Callable[[], Any]: + def run() -> Any: + start = _thread_cpu_seconds() + try: + return work() + finally: + cpu_sink.append(_thread_cpu_seconds() - start) + + return run + + def _scan_time_budget_exhausted() -> bool: remaining = _scan_time_remaining() return remaining is not None and remaining <= 0.0 @@ -538,3 +556,22 @@ def _check_always_scan_header(value: str) -> tuple[bool, str, list[dict]]: ) return True, trigger, [threat] return False, "", [] + + +async def _compiler_finditer( + compiler: Any, pattern: re.Pattern, category: str, content: str +) -> list[re.Match]: + inline_scan = category != "custom" + worker_cpu: list[float] = [] + finder = compiler.create_async_safe_finditer_matcher( + pattern, + inline_safe=inline_scan, + worker_wrapper=lambda work: _timed_in_worker(work, worker_cpu), + ) + if inline_scan: + with _record_inline_cpu(): + matches: list[re.Match] = await finder(content) + return matches + matches = await finder(content) + _record_scan_time(sum(worker_cpu) if worker_cpu else compiler.default_timeout) + return matches diff --git a/guard_core/detection_engine/compiler.py b/guard_core/detection_engine/compiler.py index d7e784f0..0a13310c 100644 --- a/guard_core/detection_engine/compiler.py +++ b/guard_core/detection_engine/compiler.py @@ -197,6 +197,7 @@ def create_safe_finditer_matcher( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Callable[[Callable[[], Any]], Callable[[], Any]] | None = None, ) -> Callable[[str], list[re.Match]]: compiled = ( pattern @@ -213,11 +214,12 @@ def inline_safe_finditer(text: str) -> list[re.Match]: return inline_safe_finditer def safe_finditer(text: str) -> list[re.Match]: + work: Callable[[], Any] = lambda: list(compiled.finditer(text)) # noqa: E731 future = shared_regex_executor().submit( - lambda: list(compiled.finditer(text)) + work if worker_wrapper is None else worker_wrapper(work) ) try: - result = future.result(timeout=match_timeout) + result: list[re.Match] = future.result(timeout=match_timeout) report_scan_success() return result except concurrent.futures.TimeoutError: @@ -234,6 +236,7 @@ def create_async_safe_finditer_matcher( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Callable[[Callable[[], Any]], Callable[[], Any]] | None = None, ) -> Callable[[str], Awaitable[list[re.Match]]]: compiled = ( pattern @@ -249,7 +252,9 @@ async def async_inline_finditer(text: str) -> list[re.Match]: return async_inline_finditer - sync_finder = self.create_safe_finditer_matcher(pattern, timeout=match_timeout) + sync_finder = self.create_safe_finditer_matcher( + pattern, timeout=match_timeout, worker_wrapper=worker_wrapper + ) async def async_safe_finditer(text: str) -> list[re.Match]: return await asyncio.to_thread(sync_finder, text) diff --git a/guard_core/handlers/_suspatterns_regex.py b/guard_core/handlers/_suspatterns_regex.py index e5ced6bd..31ad1ef9 100644 --- a/guard_core/handlers/_suspatterns_regex.py +++ b/guard_core/handlers/_suspatterns_regex.py @@ -8,11 +8,13 @@ from guard_core._utils.detection_scan import ( _EVENT_LOOP_SLICE_SECONDS, _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, - _clamp_to_scan_time_budget, + _compiler_finditer, + _record_inline_cpu, _record_scan_time, _redact_pattern_source, _scan_time_budget_exhausted, _scan_time_budget_exhausted_threat, + _timed_in_worker, _warn_scan_time_cap_reached_once, ) from guard_core.detection_engine.binary_prefix import ( @@ -346,12 +348,13 @@ def _first_accepted_regex_threat( context: str = "unknown", binary_prefix: list[int] | None = None, ) -> dict[str, Any] | None: - for match in matches: - threat = _build_regex_threat( - pattern, match, category, pattern_start, context, binary_prefix - ) - if threat: - return threat + with _record_inline_cpu(): + for match in matches: + threat = _build_regex_threat( + pattern, match, category, pattern_start, context, binary_prefix + ) + if threat: + return threat return None @@ -385,7 +388,8 @@ async def _check_regex_pattern( scan_window_matcher = _PATTERN_SCAN_WINDOW_MATCHERS.get(pattern.pattern) if scan_window_matcher is not None: - matches = scan_window_matcher(content, pattern) + with _record_inline_cpu(): + matches = scan_window_matcher(content, pattern) threat = _first_accepted_regex_threat( iter(matches), pattern, @@ -414,10 +418,7 @@ async def _check_regex_pattern( compiler = state.compiler if compiler: - safe_finder = compiler.create_async_safe_finditer_matcher( - pattern, inline_safe=category != "custom" - ) - matches = await safe_finder(content) + matches = await _compiler_finditer(compiler, pattern, category, content) timeout_threshold = 0.9 * compiler.default_timeout if not matches and time.monotonic() - pattern_start >= timeout_threshold: timeout_occurred = True @@ -464,12 +465,16 @@ async def _check_windowed_pattern( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) - timeout = _clamp_to_scan_time_budget(timeout) - future = shared_regex_executor().submit(lambda: list(finder(content))) + worker_cpu: list[float] = [] + future = shared_regex_executor().submit( + _timed_in_worker(lambda: list(finder(content)), worker_cpu) + ) try: matches = await _await_regex_future(future, timeout) report_scan_success() + _record_scan_time(sum(worker_cpu)) except concurrent.futures.TimeoutError: + _record_scan_time(timeout) logger.warning( f"Pattern timeout: {_redact_pattern_source(pattern.pattern)[:50]}..." ) @@ -532,13 +537,17 @@ async def _check_pattern_with_timeout( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) - timeout = _clamp_to_scan_time_budget(timeout) - future = shared_regex_executor().submit(pattern.search, content, search_from) + worker_cpu: list[float] = [] + future = shared_regex_executor().submit( + _timed_in_worker(lambda: pattern.search(content, search_from), worker_cpu) + ) try: match = await _await_regex_future(future, timeout) report_scan_success() + _record_scan_time(sum(worker_cpu)) return match, False except concurrent.futures.TimeoutError: + _record_scan_time(timeout) logger.warning( f"Regex timeout exceeded for pattern: " f"{_redact_pattern_source(pattern.pattern)[:50]}... " @@ -635,7 +644,6 @@ async def _check_regex_patterns( ) pattern_end = time.monotonic() - _record_scan_time(pattern_end - pattern_start) if pattern_end - slice_start >= _EVENT_LOOP_SLICE_SECONDS: await _yield_to_event_loop() slice_start = time.monotonic() diff --git a/guard_core/sync/_utils/detection_scan.py b/guard_core/sync/_utils/detection_scan.py index 8d4d433c..1f28a6dd 100644 --- a/guard_core/sync/_utils/detection_scan.py +++ b/guard_core/sync/_utils/detection_scan.py @@ -2,7 +2,7 @@ import logging import re import time -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any @@ -105,7 +105,6 @@ def _scan_time_remaining() -> float | None: _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN = "scan_time_budget_exhausted" -_MIN_CLAMPED_TIMEOUT = 0.001 _EVENT_LOOP_SLICE_SECONDS = 0.002 @@ -121,17 +120,36 @@ def _scan_time_budget_exhausted_threat(weight: float) -> dict[str, Any]: } -def _clamp_to_scan_time_budget(timeout: float) -> float: - remaining = _scan_time_remaining() - if remaining is None: - return timeout - return max(min(timeout, remaining), _MIN_CLAMPED_TIMEOUT) +def _thread_cpu_seconds() -> float: + return time.thread_time() def _record_scan_time(elapsed: float) -> None: _scan_time_spent.set(_scan_time_spent.get() + elapsed) +@contextmanager +def _record_inline_cpu() -> Iterator[None]: + start = _thread_cpu_seconds() + try: + yield + finally: + _record_scan_time(_thread_cpu_seconds() - start) + + +def _timed_in_worker( + work: Callable[[], Any], cpu_sink: list[float] +) -> Callable[[], Any]: + def run() -> Any: + start = _thread_cpu_seconds() + try: + return work() + finally: + cpu_sink.append(_thread_cpu_seconds() - start) + + return run + + def _scan_time_budget_exhausted() -> bool: remaining = _scan_time_remaining() return remaining is not None and remaining <= 0.0 @@ -536,3 +554,22 @@ def _check_always_scan_header(value: str) -> tuple[bool, str, list[dict]]: ) return True, trigger, [threat] return False, "", [] + + +def _compiler_finditer( + compiler: Any, pattern: re.Pattern, category: str, content: str +) -> list[re.Match]: + inline_scan = category != "custom" + worker_cpu: list[float] = [] + finder = compiler.create_async_safe_finditer_matcher( + pattern, + inline_safe=inline_scan, + worker_wrapper=lambda work: _timed_in_worker(work, worker_cpu), + ) + if inline_scan: + with _record_inline_cpu(): + matches: list[re.Match] = finder(content) + return matches + matches = finder(content) + _record_scan_time(sum(worker_cpu) if worker_cpu else compiler.default_timeout) + return matches diff --git a/guard_core/sync/detection_engine/compiler.py b/guard_core/sync/detection_engine/compiler.py index 64bd0455..f9034e68 100644 --- a/guard_core/sync/detection_engine/compiler.py +++ b/guard_core/sync/detection_engine/compiler.py @@ -191,6 +191,7 @@ def create_safe_finditer_matcher( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Callable[[Callable[[], Any]], Callable[[], Any]] | None = None, ) -> Callable[[str], list[re.Match]]: compiled = ( pattern @@ -207,11 +208,12 @@ def inline_safe_finditer(text: str) -> list[re.Match]: return inline_safe_finditer def safe_finditer(text: str) -> list[re.Match]: + work: Callable[[], Any] = lambda: list(compiled.finditer(text)) # noqa: E731 future = shared_regex_executor().submit( - lambda: list(compiled.finditer(text)) + work if worker_wrapper is None else worker_wrapper(work) ) try: - result = future.result(timeout=match_timeout) + result: list[re.Match] = future.result(timeout=match_timeout) report_scan_success() return result except concurrent.futures.TimeoutError: @@ -228,6 +230,7 @@ def create_async_safe_finditer_matcher( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Callable[[Callable[[], Any]], Callable[[], Any]] | None = None, ) -> Callable[[str], list[re.Match]]: compiled = ( pattern @@ -243,7 +246,9 @@ def async_inline_finditer(text: str) -> list[re.Match]: return async_inline_finditer - sync_finder = self.create_safe_finditer_matcher(pattern, timeout=match_timeout) + sync_finder = self.create_safe_finditer_matcher( + pattern, timeout=match_timeout, worker_wrapper=worker_wrapper + ) def async_safe_finditer(text: str) -> list[re.Match]: return sync_finder(text) diff --git a/guard_core/sync/handlers/_suspatterns_regex.py b/guard_core/sync/handlers/_suspatterns_regex.py index 95eed1c7..0936c2d5 100644 --- a/guard_core/sync/handlers/_suspatterns_regex.py +++ b/guard_core/sync/handlers/_suspatterns_regex.py @@ -8,11 +8,13 @@ from guard_core.sync._utils.detection_scan import ( _EVENT_LOOP_SLICE_SECONDS, _SCAN_TIME_BUDGET_EXHAUSTED_PATTERN, - _clamp_to_scan_time_budget, + _compiler_finditer, + _record_inline_cpu, _record_scan_time, _redact_pattern_source, _scan_time_budget_exhausted, _scan_time_budget_exhausted_threat, + _timed_in_worker, _warn_scan_time_cap_reached_once, ) from guard_core.sync.detection_engine.binary_prefix import ( @@ -346,12 +348,13 @@ def _first_accepted_regex_threat( context: str = "unknown", binary_prefix: list[int] | None = None, ) -> dict[str, Any] | None: - for match in matches: - threat = _build_regex_threat( - pattern, match, category, pattern_start, context, binary_prefix - ) - if threat: - return threat + with _record_inline_cpu(): + for match in matches: + threat = _build_regex_threat( + pattern, match, category, pattern_start, context, binary_prefix + ) + if threat: + return threat return None @@ -385,7 +388,8 @@ def _check_regex_pattern( scan_window_matcher = _PATTERN_SCAN_WINDOW_MATCHERS.get(pattern.pattern) if scan_window_matcher is not None: - matches = scan_window_matcher(content, pattern) + with _record_inline_cpu(): + matches = scan_window_matcher(content, pattern) threat = _first_accepted_regex_threat( iter(matches), pattern, @@ -414,10 +418,7 @@ def _check_regex_pattern( compiler = state.compiler if compiler: - safe_finder = compiler.create_async_safe_finditer_matcher( - pattern, inline_safe=category != "custom" - ) - matches = safe_finder(content) + matches = _compiler_finditer(compiler, pattern, category, content) timeout_threshold = 0.9 * compiler.default_timeout if not matches and time.monotonic() - pattern_start >= timeout_threshold: timeout_occurred = True @@ -464,12 +465,16 @@ def _check_windowed_pattern( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) - timeout = _clamp_to_scan_time_budget(timeout) - future = shared_regex_executor().submit(lambda: list(finder(content))) + worker_cpu: list[float] = [] + future = shared_regex_executor().submit( + _timed_in_worker(lambda: list(finder(content)), worker_cpu) + ) try: matches = _await_regex_future(future, timeout) report_scan_success() + _record_scan_time(sum(worker_cpu)) except concurrent.futures.TimeoutError: + _record_scan_time(timeout) logger.warning( f"Pattern timeout: {_redact_pattern_source(pattern.pattern)[:50]}..." ) @@ -532,13 +537,17 @@ def _check_pattern_with_timeout( timeout = getattr( self._config, "detection_compiler_timeout", _DEFAULT_COMPILER_TIMEOUT ) - timeout = _clamp_to_scan_time_budget(timeout) - future = shared_regex_executor().submit(pattern.search, content, search_from) + worker_cpu: list[float] = [] + future = shared_regex_executor().submit( + _timed_in_worker(lambda: pattern.search(content, search_from), worker_cpu) + ) try: match = _await_regex_future(future, timeout) report_scan_success() + _record_scan_time(sum(worker_cpu)) return match, False except concurrent.futures.TimeoutError: + _record_scan_time(timeout) logger.warning( f"Regex timeout exceeded for pattern: " f"{_redact_pattern_source(pattern.pattern)[:50]}... " @@ -635,7 +644,6 @@ def _check_regex_patterns( ) pattern_end = time.monotonic() - _record_scan_time(pattern_end - pattern_start) if pattern_end - slice_start >= _EVENT_LOOP_SLICE_SECONDS: _yield_to_event_loop() slice_start = time.monotonic() diff --git a/tests/test_scan_time_budget.py b/tests/test_scan_time_budget.py index 26c33b1f..a331662f 100644 --- a/tests/test_scan_time_budget.py +++ b/tests/test_scan_time_budget.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import AsyncGenerator, Iterator from typing import Any from unittest.mock import patch @@ -35,7 +36,7 @@ def __init__(self, step: float) -> None: self.now = 0.0 self.step = step - def monotonic(self) -> float: + def cpu_seconds(self) -> float: self.now += self.step return self.now @@ -43,8 +44,7 @@ def monotonic(self) -> float: @pytest.fixture def fake_regex_clock() -> Iterator[_FakeClock]: clock = _FakeClock(step=1.0) - with patch("guard_core.handlers._suspatterns_regex.time") as fake_time: - fake_time.monotonic = clock.monotonic + with patch.object(detection_scan, "_thread_cpu_seconds", clock.cpu_seconds): yield clock @@ -183,3 +183,37 @@ def test_threats_message_appends_the_first_later_timeout() -> None: assert detection_scan._build_threats_message([timeout, regex]) == ( "Pattern exceeded scan time budget: 'slow'" ) + + +async def test_wall_slow_but_cpu_cheap_scan_does_not_exhaust_the_budget() -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + real = manager._check_regex_pattern + + async def slow_wall(*args: Any, **kwargs: Any) -> Any: + await asyncio.sleep(0.01) + return await real(*args, **kwargs) + + with ( + patch.object(manager, "_check_regex_pattern", slow_wall), + _scan_value_budget(512, max_scan_seconds=0.1), + ): + result = await manager.detect("hello world", "127.0.0.1", "query_param:q") + + assert all(t["pattern"] != "scan_time_budget_exhausted" for t in result["threats"]) + assert result["is_threat"] is False + + +async def test_worker_scan_cpu_is_recorded_from_the_worker_thread( + fake_regex_clock: _FakeClock, +) -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + await manager.add_pattern("zzzworker[a-z]+q") + + with _scan_value_budget(512, max_scan_seconds=1000.0): + await manager.detect("hello", "127.0.0.1", "query_param:q") + remaining = detection_scan._scan_time_remaining() + + assert remaining is not None + assert remaining < 1000.0 diff --git a/tests/test_shared_regex_executor.py b/tests/test_shared_regex_executor.py index 989ce723..f43c8654 100644 --- a/tests/test_shared_regex_executor.py +++ b/tests/test_shared_regex_executor.py @@ -146,9 +146,12 @@ def _tracking( pattern: Any, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Any: calls.append(pattern) - return real(pattern, timeout, inline_safe=inline_safe) + return real( + pattern, timeout, inline_safe=inline_safe, worker_wrapper=worker_wrapper + ) monkeypatch.setattr( fresh_manager._compiler, @@ -176,9 +179,12 @@ def _tracking( pattern: Any, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Any: calls.append(pattern) - return real(pattern, timeout, inline_safe=inline_safe) + return real( + pattern, timeout, inline_safe=inline_safe, worker_wrapper=worker_wrapper + ) monkeypatch.setattr( fresh_manager._compiler, diff --git a/tests/test_sus_patterns/test_custom_redos_hardening.py b/tests/test_sus_patterns/test_custom_redos_hardening.py index a09f3894..e256c1bc 100644 --- a/tests/test_sus_patterns/test_custom_redos_hardening.py +++ b/tests/test_sus_patterns/test_custom_redos_hardening.py @@ -3,6 +3,7 @@ import sys import time from collections.abc import Awaitable, Callable +from typing import Any from unittest.mock import patch import pytest @@ -924,9 +925,16 @@ def _spy( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Callable[[str], Awaitable[list[re.Match]]]: captured.append(inline_safe) - return original(self, pattern, timeout=timeout, inline_safe=inline_safe) + return original( + self, + pattern, + timeout=timeout, + inline_safe=inline_safe, + worker_wrapper=worker_wrapper, + ) with patch.object(PatternCompiler, "create_async_safe_finditer_matcher", _spy): result = await sus_patterns_handler.detect( @@ -946,9 +954,16 @@ def _spy( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Callable[[str], Awaitable[list[re.Match]]]: captured.append(inline_safe) - return original(self, pattern, timeout=timeout, inline_safe=inline_safe) + return original( + self, + pattern, + timeout=timeout, + inline_safe=inline_safe, + worker_wrapper=worker_wrapper, + ) with patch.object(PatternCompiler, "create_async_safe_finditer_matcher", _spy): result = await sus_patterns_handler.detect( diff --git a/tests/test_sync/test_scan_time_budget.py b/tests/test_sync/test_scan_time_budget.py index 84f65c6d..4d0a071e 100644 --- a/tests/test_sync/test_scan_time_budget.py +++ b/tests/test_sync/test_scan_time_budget.py @@ -1,3 +1,4 @@ +import time from collections.abc import Generator, Iterator from typing import Any from unittest.mock import patch @@ -35,7 +36,7 @@ def __init__(self, step: float) -> None: self.now = 0.0 self.step = step - def monotonic(self) -> float: + def cpu_seconds(self) -> float: self.now += self.step return self.now @@ -43,8 +44,7 @@ def monotonic(self) -> float: @pytest.fixture def fake_regex_clock() -> Iterator[_FakeClock]: clock = _FakeClock(step=1.0) - with patch("guard_core.sync.handlers._suspatterns_regex.time") as fake_time: - fake_time.monotonic = clock.monotonic + with patch.object(detection_scan, "_thread_cpu_seconds", clock.cpu_seconds): yield clock @@ -185,3 +185,37 @@ def test_threats_message_appends_the_first_later_timeout() -> None: assert detection_scan._build_threats_message([timeout, regex]) == ( "Pattern exceeded scan time budget: 'slow'" ) + + +def test_wall_slow_but_cpu_cheap_scan_does_not_exhaust_the_budget() -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + real = manager._check_regex_pattern + + def slow_wall(*args: Any, **kwargs: Any) -> Any: + time.sleep(0.01) + return real(*args, **kwargs) + + with ( + patch.object(manager, "_check_regex_pattern", slow_wall), + _scan_value_budget(512, max_scan_seconds=0.1), + ): + result = manager.detect("hello world", "127.0.0.1", "query_param:q") + + assert all(t["pattern"] != "scan_time_budget_exhausted" for t in result["threats"]) + assert result["is_threat"] is False + + +def test_worker_scan_cpu_is_recorded_from_the_worker_thread( + fake_regex_clock: _FakeClock, +) -> None: + manager = SusPatternsManager() + manager.configure(SecurityConfig()) + manager.add_pattern("zzzworker[a-z]+q") + + with _scan_value_budget(512, max_scan_seconds=1000.0): + manager.detect("hello", "127.0.0.1", "query_param:q") + remaining = detection_scan._scan_time_remaining() + + assert remaining is not None + assert remaining < 1000.0 diff --git a/tests/test_sync/test_shared_regex_executor.py b/tests/test_sync/test_shared_regex_executor.py index a24d1e2c..a342d86f 100644 --- a/tests/test_sync/test_shared_regex_executor.py +++ b/tests/test_sync/test_shared_regex_executor.py @@ -146,9 +146,12 @@ def _tracking( pattern: Any, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Any: calls.append(pattern) - return real(pattern, timeout, inline_safe=inline_safe) + return real( + pattern, timeout, inline_safe=inline_safe, worker_wrapper=worker_wrapper + ) monkeypatch.setattr( fresh_manager._compiler, @@ -176,9 +179,12 @@ def _tracking( pattern: Any, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Any: calls.append(pattern) - return real(pattern, timeout, inline_safe=inline_safe) + return real( + pattern, timeout, inline_safe=inline_safe, worker_wrapper=worker_wrapper + ) monkeypatch.setattr( fresh_manager._compiler, diff --git a/tests/test_sync/test_sus_patterns/test_custom_redos_hardening.py b/tests/test_sync/test_sus_patterns/test_custom_redos_hardening.py index ceed5ff2..8169bb79 100644 --- a/tests/test_sync/test_sus_patterns/test_custom_redos_hardening.py +++ b/tests/test_sync/test_sus_patterns/test_custom_redos_hardening.py @@ -3,6 +3,7 @@ import sys import time from collections.abc import Callable +from typing import Any from unittest.mock import patch import pytest @@ -924,9 +925,16 @@ def _spy( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Callable[[str], list[re.Match]]: captured.append(inline_safe) - return original(self, pattern, timeout=timeout, inline_safe=inline_safe) + return original( + self, + pattern, + timeout=timeout, + inline_safe=inline_safe, + worker_wrapper=worker_wrapper, + ) with patch.object(PatternCompiler, "create_async_safe_finditer_matcher", _spy): result = sus_patterns_handler.detect( @@ -946,9 +954,16 @@ def _spy( pattern: str | re.Pattern, timeout: float | None = None, inline_safe: bool = False, + worker_wrapper: Any = None, ) -> Callable[[str], list[re.Match]]: captured.append(inline_safe) - return original(self, pattern, timeout=timeout, inline_safe=inline_safe) + return original( + self, + pattern, + timeout=timeout, + inline_safe=inline_safe, + worker_wrapper=worker_wrapper, + ) with patch.object(PatternCompiler, "create_async_safe_finditer_matcher", _spy): result = sus_patterns_handler.detect( From 41bd6b5186d6179e22de65fedd37e858d3c3db96 Mon Sep 17 00:00:00 2001 From: Renn F Date: Thu, 15 Oct 2026 06:39:52 +0200 Subject: [PATCH 4/5] test(detection): cover the semantic header opt-out across every Cookie pair --- tests/test_semantic_excluded_headers.py | 13 +++++++++++++ tests/test_sync/test_semantic_excluded_headers.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/test_semantic_excluded_headers.py b/tests/test_semantic_excluded_headers.py index 48e7caca..df126dc0 100644 --- a/tests/test_semantic_excluded_headers.py +++ b/tests/test_semantic_excluded_headers.py @@ -78,6 +78,19 @@ async def test_listed_header_is_still_scanned_by_regex() -> None: assert "Semantic attack" not in result.trigger_info +async def test_listed_header_opt_out_applies_to_every_cookie_pair() -> None: + cookie = f"theme=dark; {_JWT_COOKIE}; lang=en" + request = MockGuardRequest(headers={"cookie": cookie}) + + flagged = await detect_penetration_attempt(request, _config()) + exempt = await detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert flagged.is_threat is True + assert exempt.is_threat is False + + async def test_route_config_set_is_merged() -> None: request = MockGuardRequest(headers={"cookie": _JWT_COOKIE}) route_config = RouteConfig() diff --git a/tests/test_sync/test_semantic_excluded_headers.py b/tests/test_sync/test_semantic_excluded_headers.py index cb6d9cf1..3fd8a129 100644 --- a/tests/test_sync/test_semantic_excluded_headers.py +++ b/tests/test_sync/test_semantic_excluded_headers.py @@ -80,6 +80,19 @@ def test_listed_header_is_still_scanned_by_regex() -> None: assert "Semantic attack" not in result.trigger_info +def test_listed_header_opt_out_applies_to_every_cookie_pair() -> None: + cookie = f"theme=dark; {_JWT_COOKIE}; lang=en" + request = SyncMockGuardRequest(headers={"cookie": cookie}) + + flagged = detect_penetration_attempt(request, _config()) + exempt = detect_penetration_attempt( + request, _config(excluded_semantic_detection_headers={"cookie"}) + ) + + assert flagged.is_threat is True + assert exempt.is_threat is False + + def test_route_config_set_is_merged() -> None: request = SyncMockGuardRequest(headers={"cookie": _JWT_COOKIE}) route_config = RouteConfig() From 392b7ef6549537183b67719c177ad24efd00bd5a Mon Sep 17 00:00:00 2001 From: Renn F Date: Thu, 15 Oct 2026 07:38:38 +0200 Subject: [PATCH 5/5] test(live-smoke): cover detection_max_scan_seconds and excluded_semantic_detection_headers --- .../live_smoke/scenarios/detection_engine.py | 31 +++++++++ .../framework_gaps_detection_limits.py | 69 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/tests/live_smoke/scenarios/detection_engine.py b/tests/live_smoke/scenarios/detection_engine.py index 9b8ec626..6c1a0058 100644 --- a/tests/live_smoke/scenarios/detection_engine.py +++ b/tests/live_smoke/scenarios/detection_engine.py @@ -505,3 +505,34 @@ def pattern_validation_cache_persists_cost_verdicts( error_lines = ctx.stack.logs.lines_since(mark, kind="Failed to") assert not error_lines, f"unexpected 'Failed to' error lines: {error_lines}" + + +_JWT_COOKIE = ( + "session=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ." + "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" +) + + +@scenario( + covers={"excluded_semantic_detection_headers"}, + config={ + **_BASE, + "detection_semantic_threshold": 0.3, + "excluded_semantic_detection_headers": ["cookie"], + }, +) +def excluded_semantic_detection_headers_skip_only_semantic_scoring( + ctx: ScenarioContext, +) -> None: + listed = ctx.client.get("/basic/ip", headers={"cookie": _JWT_COOKIE}) + assert listed.status_code == 200, ( + "excluded_semantic_detection_headers=['cookie'] still flagged a JWT " + f"session cookie semantically: {listed.status_code}" + ) + + unlisted = ctx.client.get("/basic/ip", headers={"x-session": _JWT_COOKIE}) + assert unlisted.status_code == 400, ( + "a header not in excluded_semantic_detection_headers lost its " + f"semantic analysis: {unlisted.status_code}" + ) diff --git a/tests/live_smoke/scenarios/framework_gaps_detection_limits.py b/tests/live_smoke/scenarios/framework_gaps_detection_limits.py index 4fb85468..2aa9b6c0 100644 --- a/tests/live_smoke/scenarios/framework_gaps_detection_limits.py +++ b/tests/live_smoke/scenarios/framework_gaps_detection_limits.py @@ -177,3 +177,72 @@ def sync_body_read_max_concurrent_serializes_reads(ctx: ScenarioContext) -> None f"max_concurrent=1 should serialize the two reads; B started at " f"{b_start:.2f}s, before A finished at {a_end:.2f}s: {events}" ) + + +_SCAN_BUDGET_PROBE = """ +import asyncio +import logging +import types + +logging.disable(logging.CRITICAL) + +from guard_core._utils import detection_scan +from guard_core.models import SecurityConfig +from guard_core.utils import detect_penetration_attempt + + +class _Clock: + now = 0.0 + + @classmethod + def tick(cls): + cls.now += 1.0 + return cls.now + + +detection_scan._thread_cpu_seconds = _Clock.tick + + +class _Request: + headers = {} + url_path = "/" + url_scheme = "http" + url_full = "http://smoke/" + method = "GET" + client_host = "203.0.113.9" + scope = {} + state = types.SimpleNamespace() + query_params = {"q": "select name from the users table where the quick brown fox"} + + async def body(self): + return b"" + + +async def main(): + config = SecurityConfig(detection_max_scan_seconds=0.1) + result = await detect_penetration_attempt(_Request(), config) + print(f"SMOKE_SCAN_BUDGET_PROBE threat={result.is_threat} " + f"info={result.trigger_info!r}", flush=True) + + +asyncio.run(main()) +""" + + +@scenario( + covers={"detection_max_scan_seconds"}, + config={**COMPILER_TIMEOUT_CONFIG, "detection_max_scan_seconds": 0.1}, +) +def detection_max_scan_seconds_flags_a_request_that_exhausts_the_budget( + ctx: ScenarioContext, +) -> None: + result = run_in_app_container(_SCAN_BUDGET_PROBE, timeout=20.0) + output = result.stdout + result.stderr + + assert "SMOKE_SCAN_BUDGET_PROBE" in output, ( + f"the detection_max_scan_seconds probe did not complete; output: {output!r}" + ) + assert "threat=True" in output and "scan_time_budget_exhausted" in output, ( + "detection_max_scan_seconds=0.1 did not flag a request whose regex CPU " + f"time exceeded the budget; probe output: {output!r}" + )