diff --git a/AGENTS.md b/AGENTS.md index 4b089d4b..93353eb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -168,19 +168,20 @@ def __init__(self, name="MyDetector", config=MyDetectorConfig()): self._register_persistency(self.persistency) # must be last ``` -**2. Preserve `config.persist` across `set_configuration()` rebuilds:** +**2. Write only your outputs in `set_configuration()` — never rebuild `self.config`:** -`set_configuration()` replaces `self.config` via `from_dict()`, which produces a config with no `persist` key — silently dropping the user's persist settings. Save and restore it: +`set_configuration()` must not reassign `self.config` (e.g. via `from_dict()`). Write only what the configure phase produced, then flip `auto_config` off: ```python def set_configuration(self) -> None: - old_persist = self.config.persist - # ... build config_dict, call from_dict() ... - self.config = MyDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist + variables = {...} # whatever the configure phase decided + self.config.events = generate_events_config(variables, self.name) + self.config.auto_config = False ``` -Omitting either step means a `persist:` block in the YAML is silently ignored with no error. +(`EventSequenceDetector` additionally writes `self.config.fixed_window_size`, since its configure phase picks a window length rather than a variable selection.) + +Every other field — `persist`, `auto_config_params`, any detector-specific param — is operator input and is left untouched by construction, since nothing here reassigns `self.config`. Rebuilding the config wholesale is what used to drop `persist` (and everything else) silently. ## Code Quality diff --git a/docs/detectors.md b/docs/detectors.md index e4bbe4d7..e369be35 100644 --- a/docs/detectors.md +++ b/docs/detectors.md @@ -200,7 +200,10 @@ def configure(self, input_): ) ``` -The `set_configuration()` method queries the tracker results and generates the final config: +The `set_configuration()` method queries the tracker results and writes the +final `events` block. It touches nothing else on the config — everything the +operator set under `params` or `auto_config_params` must survive untouched, so +`set_configuration` never rebuilds the config from scratch: ```python def set_configuration(self): @@ -209,12 +212,8 @@ def set_configuration(self): stable_vars = tracker.get_features_by_classification("STABLE") variables[event_id] = stable_vars - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - self.config = MyDetectorConfig.from_dict(config_dict, self.name) + self.config.events = generate_events_config(variables, self.name) + self.config.auto_config = False ``` ### Full lifecycle with auto-configuration @@ -228,78 +227,171 @@ def set_configuration(self): When `auto_config` is `False`, steps 1 and 2 are skipped entirely. - -### Stability segmentation (optional) - -Stability classification splits a variable's change history into four segments and -compares each segment's rate of change against a threshold. By default the segments -are **equal-count**: each holds the same number of observations, regardless of how much -time they cover. For bursty log sources that is misleading — a variable that changed -constantly during a quiet night and then went silent under a flood of daytime traffic -looks stable, because the flood supplies enough samples to dominate the later segments. - -Setting `stability_segmentation: time` switches the segmentation to **equal-duration** cuts -of the observed time span, so each segment covers the same amount of wall-clock time. The -detector then needs an event time per record, which it reads from the log's named -variables (`logFormatVariables`, i.e. the fields declared in the parser's `log_format`) -under the name given by `timestamp_variable`. - -These three parameters live on every `VariableDetector` subclass (`NewValueDetector`, +That distinction is visible in the config. A detector's settings live in two +blocks: + +* **`auto_config_params`** — inputs *to* the configure phase. They pick which + variables the phase selects and are read only while `auto_config` is `True`. +* **`params`** — operational settings, read during training and detection on + every run. + +The configure phase writes its results into the top-level `events` block (and, +for `EventSequenceDetector`, into `fixed_window_size`) and then sets +`auto_config` to `False`. It never modifies either input block, so a config can +be rerun with `auto_config: False` and reproduce the same detector. + +Both `auto_config` and `Component.configure()` are declared on the shared base, +so `auto_config_params` is declared there too — on `BasicConfig`, beside +`auto_config` — rather than on the detector config alone. Detectors are the only +component type with a real configure phase today, so they are the only ones that +narrow the block with fields; parsers and alert aggregators inherit it empty, and +an empty block is omitted from the serialized config, so their YAML is unaffected. +A component type that grows a configure phase later subclasses `AutoConfigParams` +and overrides the field, exactly as the variable, combo and sequence detector +families do. + + +### Stability classification (optional) + +Stability classification decides whether a variable's change history counts as +`STABLE` by running one or more classification methods against it and combining +their verdicts. There are four independent methods, over two primitives and two +axes: + +| method | what it thresholds | axis | +|---|---|---| +| `index` | segment-mean thresholds | equal-count boundaries | +| `time` | segment-mean thresholds | equal-duration boundaries | +| `slope_index` | change centroid vs. `slope_threshold` | index positions | +| `slope_time` | change centroid vs. `slope_threshold` | normalized timestamps | + +Any subset of the four may be enabled, and any single one may stand alone. The +default — `index` alone — is the historical behaviour: each segment's mean rate +of change is compared against a threshold, and the segments are **equal-count**: +each holds the same number of observations, regardless of how much time they +cover. For bursty log sources that is misleading — a variable that changed +constantly during a quiet night and then went silent under a flood of daytime +traffic looks stable, because the flood supplies enough samples to dominate the +later segments. Enabling `time` cuts the same four segments at **equal +durations** instead, so each segment covers the same amount of wall-clock time; +the detector then needs an event time per record, which it reads from the log's +named variables (`logFormatVariables`, i.e. the fields declared in the parser's +`log_format`) under the name given by `timestamp_variable`. `slope_index` and +`slope_time` ask a different question — whether the change centroid sits early +or late in the series — on the index axis and the time axis respectively. + +These parameters live on every `VariableDetector` subclass (`NewValueDetector`, `NewValueComboDetector`, `ValueRangeDetector`, `CharsetDetector`, `BigramDetector`, …) -and go in the detector's top-level `params` block: +and go in the detector's `auto_config_params` block — they are inputs to the +auto-configuration phase, read only while `auto_config` is `True`, and never +consulted at detection time. ```yaml detectors: NewValueDetector: method_type: new_value_detector auto_config: True - params: - stability_segmentation: time - timestamp_variable: Time # a field name from the parser's log_format - timestamp_format: "%y%m%d %H%M%S" # optional; omit to auto-detect + auto_config_params: + use_stable_vars: True + use_static_vars: True + classification: + index: True # segment-mean thresholds, equal-count cuts + time: False # segment-mean thresholds, equal-duration cuts + slope_index: False # change centroid over index positions + slope_time: False # change centroid over normalized time + slope_threshold: -0.05 # shared by both slope methods + decision: consensus # consensus | majority + timestamp_variable: Time + timestamp_format: "%y%m%d %H%M%S" ``` -Setting `stability_segmentation: both` runs *both* segmentations and calls the variable -stable only when each one does. Neither segmentation subsumes the other — a variable that -churns in a burst and then settles is unstable by count but stable by time, and one whose -late churn is buried under a dense tail of repeats is the reverse — so `both` is strictly -stricter than either. Use it when a false "stable" is more costly than a missed one; use -`time` when the point is specifically to forgive early churn on a bursty source. +Defaults reproduce the historical behaviour exactly: `index: True`, the other +three `False`, `decision: consensus`, `slope_threshold: -0.05`. A config that +sets nothing under `classification` classifies identically to before this change. + +#### The decision rule + +When more than one method is enabled, `decision` picks how their verdicts +combine. `consensus` requires every enabled method to return stable; `majority` +requires strictly more than half of them to. + +| enabled | `consensus` needs | `majority` needs | differ? | +|---|---|---|---| +| 1 | 1/1 | 1/1 | no | +| 2 | 2/2 | 2/2 (a 1–1 tie is UNSTABLE) | no | +| 3 | 3/3 | 2/3 | yes | +| 4 | 4/4 | 3/4 (a 2–2 tie is UNSTABLE) | yes | + +Ties resolve to UNSTABLE. That keeps `majority` from ever being more lenient +than a coin-flip, and makes it collapse onto `consensus` at one and two enabled +methods — turning a third method on is the only place the rule starts to matter. + +**All four methods false is a config error**, rejected by a pydantic validator. +It is not a harmless no-op: classification decides `INSUFFICIENT_DATA`, +`STATIC` and `RANDOM` before any method is consulted, so a method-less config +would silently classify every remaining variable `STABLE`. #### Fields +All of these live in the detector's `auto_config_params` block. + | Field | Type | Default | Description | |---|---|---|---| -| `stability_segmentation` | `"count" \| "time" \| "both"` | `"count"` | How to cut the change history into segments. `count` uses equal sample counts; `time` uses equal time spans; `both` requires the variable to be stable under each. With `count` the other two fields are ignored and no timestamps are recorded. | -| `timestamp_variable` | `str \| null` | `null` | Name of the field in `logFormatVariables` holding the record's event time. Required for `time` and `both` to have any effect. Only named log-format fields are consulted — never the positional `variables` list. | +| `use_stable_vars` | `bool` | `true` | Include variables classified `STABLE` in the generated configuration. | +| `use_static_vars` | `bool` | `true` | Include variables classified `STATIC`. Defaults to `false` on `NewValueComboDetector`. | +| `classification` | `ClassificationMethods` | see below | Which classification methods run and how their verdicts combine. | +| `timestamp_variable` | `str \| null` | `null` | Name of the field in `logFormatVariables` holding the record's event time. Required for `time` and `slope_time` to have any effect. Only named log-format fields are consulted — never the positional `variables` list. | | `timestamp_format` | `str \| null` | `null` | Explicit [`strftime`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) pattern for parsing that field. When unset, `TimeFormatHandler` auto-detects the format (ISO 8601, Apache, syslog, numeric epoch seconds/milliseconds, and other common layouts). | Set `timestamp_format` when the source uses a layout the auto-detection does not know. The HDFS loghub corpus, for example, stamps records as `081109 203615`, which only parses with an explicit `"%y%m%d %H%M%S"`. +`classification`'s six fields: + +| Field | Type | Default | Description | +|---|---|---|---| +| `index` | `bool` | `true` | Segment-mean thresholds, equal-count boundaries. | +| `time` | `bool` | `false` | Segment-mean thresholds, equal-duration boundaries. Needs `timestamp_variable`. | +| `slope_index` | `bool` | `false` | Change centroid vs. `slope_threshold`, measured on index positions. | +| `slope_time` | `bool` | `false` | Change centroid vs. `slope_threshold`, measured on normalized timestamps. Needs `timestamp_variable`. | +| `slope_threshold` | `float` | `-0.05` | The change-centroid cut-off both slope methods compare against, on a shared `[-0.5, +0.5]` scale. A variable passes when its centroid is at or below this value. | +| `decision` | `"consensus" \| "majority"` | `"consensus"` | How verdicts from more than one enabled method combine; see above. | + #### Fallback behaviour -Time-aware segmentation is best-effort and never fails a run: +Time-aware classification is best-effort and never fails a run: -* If `stability_segmentation` is not `count` but `timestamp_variable` is unset, or the named field - is absent from a record, or its value cannot be parsed, the detector logs a +* If `time` or `slope_time` is enabled but `timestamp_variable` is unset, or the named + field is absent from a record, or its value cannot be parsed, the detector logs a **single** warning (once per detector, so a bad config cannot flood the log) and - falls back to count-based segmentation. + falls back to the index axis. * If timestamps stop lining up with the recorded observations, or the observed time - span is zero, or they arrive out of order, the classifier silently falls back to - count-based segmentation for that variable. -* Under `both`, any of the fallbacks above make the time pass reuse the count boundaries, - so the mode degrades to plain `count` rather than to an unconditional pass. + span is zero, or they arrive out of order, `time` silently reuses the equal-index + cuts, and `slope_time` computes its centroid on the index axis instead — it + degrades to `slope_index`. +* Under `majority`, a fallen-back method still casts its own vote: if `slope_index` + and `slope_time` are both enabled and timestamps are unusable, both entries compute + the same index-axis centroid, and that verdict carries two of the votes rather than + one. This is deliberate — dropping a fallen-back method from the vote would change + the enabled count from variable to variable and make `majority` mean something + different for each one. The reason string names the axis each slope actually used, + so a doubled vote is visible in the note. +* The same doubling applies to the segment-threshold pair: if `index` and `time` are + both enabled and timestamps are unusable, `time` silently reuses the same equal-count + cuts as `index`, so an identical verdict again carries two votes under `majority` + rather than one. Unlike the slope pair, the reason string does not surface this — + each entry is still labelled by its configured method name (`index` or `time`), not + by the axis it actually used, so a doubled segment-pair vote is invisible in the note. In every fallback case classification still runs and produces a result — only the -segmentation rule changes back to the default. +axis behind it changes back to index. A segment with no observations in it is *not* a fallback: it scores a mean of 0.0, because nothing observed means nothing changed. Equal-duration cuts of a bursty variable leave such segments routinely, so `time` on its own is lenient towards a -burst of churn followed by silence. Use `both` when that leniency matters — the -count pass keeps every segment populated. +burst of churn followed by silence. Enable `index` and `time` together when that +leniency matters — the index pass keeps every segment populated. ### Saving state (persist) diff --git a/docs/detectors/charset.md b/docs/detectors/charset.md index 3ed4a6e9..63c71689 100644 --- a/docs/detectors/charset.md +++ b/docs/detectors/charset.md @@ -1,4 +1,4 @@ -# New Value Detector +# Charset Detector The Charset Detector raises alerts when previously unseen characters appear in configured fields. It is useful to detect novelty, configuration drift, or the appearance of new actors in the environment. diff --git a/docs/detectors/combo.md b/docs/detectors/combo.md index 6ae99e46..6bbeaa70 100644 --- a/docs/detectors/combo.md +++ b/docs/detectors/combo.md @@ -17,7 +17,7 @@ detectors: NewValueComboDetector: method_type: new_value_combo_detector auto_config: False - params: + auto_config_params: max_combo_size: 3 events: 1: diff --git a/docs/detectors/ecvc_detector.md b/docs/detectors/ecvc_detector.md index f979618c..feb39782 100644 --- a/docs/detectors/ecvc_detector.md +++ b/docs/detectors/ecvc_detector.md @@ -11,6 +11,8 @@ The Event Count Vector Clustering Detector (ECVC) detects anomalies by calculati A count vector is form by counting the number of appearance of each event ID in a sequence of a specific window size. +Count vectors learned during training are stored via [persistency](../auxiliar/persistency.md), so a trained model can be saved and restored with a `persist:` block. A count vector is only comparable within the window it was counted over, so restoring state at a different `window_size` logs a warning — the restored vectors cannot match and every window would alert. + ## Configuration example diff --git a/docs/detectors/event_sequence.md b/docs/detectors/event_sequence.md index ab33fa88..8759dfff 100644 --- a/docs/detectors/event_sequence.md +++ b/docs/detectors/event_sequence.md @@ -44,14 +44,19 @@ detectors: method_type: event_sequence_detector auto_config: True data_use_configure: 500 - params: + auto_config_params: min_window_size: 2 max_window_size: 10 ``` | Parameter | Default | Description | |---|---|---| -| `fixed_window_size` | `None` | Length of the sliding event-ID window. Overrides `min_window_size`/`max_window_size` and skips auto configuration. Auto configuration writes its own choice here. While it is `None` the detector neither trains nor alerts. Must be `>= 1`. | +| `fixed_window_size` | `None` | Length of the sliding event-ID window. Overrides the `auto_config_params` window range and skips auto configuration. Auto configuration writes its own choice here. While it is `None` the detector neither trains nor alerts. Must be `>= 1`. | + +#### `auto_config_params` + +| Field | Default | Description | +|---|---|---| | `min_window_size` | `2` | Shortest window length tried during auto configuration. Must be `>= 1`. | | `max_window_size` | `10` | Longest window length tried during auto configuration. Must be `>= min_window_size`. | diff --git a/docs/detectors/scvs_detector.md b/docs/detectors/scvs_detector.md index 8935516e..6025f700 100644 --- a/docs/detectors/scvs_detector.md +++ b/docs/detectors/scvs_detector.md @@ -11,6 +11,8 @@ The Sequence Count Vector Set Detector (SCVS) detects anomalies by finding count A count vector is formed by counting the number of appearance of each event ID in a sequence of a specific window size. +Count vectors learned during training are stored via [persistency](../auxiliar/persistency.md), so a trained model can be saved and restored with a `persist:` block. A count vector is only comparable within the window it was counted over, so restoring state at a different `window_size` logs a warning — the restored vectors cannot match and every window would alert. + ## Configuration example diff --git a/docs/development.md b/docs/development.md index a65a0317..e8293958 100644 --- a/docs/development.md +++ b/docs/development.md @@ -41,3 +41,70 @@ In order to run the tests run the following command. The `dev` group already inc ```bash uv run --dev pytest ``` + +## Write testable code snippets for the documentation + +Code examples in the docs are not pasted inline. They live as standalone Python +files under `docs/examples/`, mirrored by category (`docs/examples/parsers/`, +`docs/examples/detectors/`), and are pulled into the Markdown pages via +[`pymdownx.snippets`](https://facelessuser.github.io/pymdown-extensions/extensions/snippets/). +This way every snippet in the docs is an actual `.py` file that gets executed by +the test suite, so a broken example fails CI instead of silently shipping. + +**1. Add the snippet file.** Put your example under `docs/examples//`. +By convention the filename matches its documentation page (`charset.md` → +`docs/examples/detectors/charset.py`). Wrap the part you want to show in section +markers: + +```python +# ;--8<-- [start:basic] +from detectmatelibrary.parsers.logbatcher import LogBatcherParser, LogBatcherParserConfig +# ... +# ;--8<-- [end:basic] +``` + +**2. Include it in the `.md` page.** Paths are relative to the repo root +(`base_path` is set to `.`). Reference the section by name: + +````markdown +```python +;--8<-- "docs/examples/parsers/logbatcher_parser.py:basic" +``` +```` + +You can also include the whole file by dropping the `:section` suffix +(`--8<-- "docs/examples/parsers/template_tree_matcher.py"`), but section markers +are the norm. Because `check_paths: true` is set, the build aborts if the file or +marker doesn't exist — a missing snippet is caught at build time. + +**3. Make sure it's testable.** The test (`tests/test_docs/test_doc_examples.py`) +globs every `.py` under `docs/examples/` and runs each one as a script via +`runpy.run_path(..., run_name="__main__")`. There is no plugin and no assert +requirement: a snippet passes as long as it runs standalone without raising. If +your example needs something unavailable in CI (e.g. an API key), comment out +those calls rather than letting them fail. Run the snippet tests together with +the rest of the suite: + +```bash +uv run --dev pytest +``` + + +## Render and verify the documentation + +Build the static site: + +```bash +uv run --dev mkdocs build +``` + +For a live local preview while editing: + +```bash +uv run --dev mkdocs serve +``` + +`mkdocs` comes in transitively via `mike` in the `dev` group, so `--dev` is +required. There is no `--strict` mode configured; the hard check on the docs is +`check_paths: true` from `pymdownx.snippets`, which fails the build on a missing +snippet or marker. diff --git a/docs/diagrams.drawio b/docs/diagrams.drawio index 61c4bf68..3444a1ca 100644 --- a/docs/diagrams.drawio +++ b/docs/diagrams.drawio @@ -1,108 +1,154 @@ - - + + - - + + - - + + - - + + - - + + - - + + - + + + + + + + - - - - - - + + - + - - - - - - + + - - + + + + + - - + + - - + + - + + + + - - + + - + - - + + - + - - + + - - + + + + + - - + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - + + - + - - - - - - + + + + + + + + + + + + + + - + @@ -215,25 +261,25 @@ - + - + - + - + - + @@ -242,8 +288,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -280,16 +428,16 @@ - + - + - + diff --git a/docs/examples/detectors/combo.py b/docs/examples/detectors/combo.py index e88f8941..2f24d2b2 100644 --- a/docs/examples/detectors/combo.py +++ b/docs/examples/detectors/combo.py @@ -7,7 +7,7 @@ "NewValueTest": { "method_type": "new_value_combo_detector", "auto_config": False, - "params": { + "auto_config_params": { "max_combo_size": 4 }, "events": { diff --git a/docs/examples/others/federation.py b/docs/examples/others/federation.py new file mode 100644 index 00000000..8e21ffa2 --- /dev/null +++ b/docs/examples/others/federation.py @@ -0,0 +1,88 @@ + +# --8<-- [start:example_1] +from detectmatelibrary.common.core import CoreComponent +import struct + + +class NewComponent(CoreComponent): # Inherent from CoreComponent + def __init__(self, elems): + self.elems = elems + super().__init__(name="FedExample") + + def aggregate_strategy(self, components): + final_list = [] + for component in components: + final_list.extend(component.elems) + + final_list = list(set(final_list)) + for component in components: + component.elems = final_list + + def to_binary(self): + return struct.pack(f">{len(self.elems)}h", *self.elems) + + def from_binary(self, binary): + num_ints = len(binary) // 2 + elems = list(struct.unpack(f">{num_ints}h", binary)) + return NewComponent(elems=elems) + + +# --8<-- [end:example_1] +# --8<-- [start:example_2] +detector1 = NewComponent([1, 2, 3]) +detector2 = NewComponent([4, 5]) +detector3 = NewComponent([6]) + +detector1 + detector2 + detector3 + +detector2.aggregate() # Detector 2 is used as centralize node + +print("Dectector 3", detector3.elems) # All detectors have been updated + +# --8<-- [end:example_2] + +# --8<-- [start:example_3] +detector1 = NewComponent([1, 2, 3]) +detector2 = NewComponent([4, 5]) +detector3 = NewComponent([6]) + +detector1 + detector2 + detector3 + +detector2.aggregate() # Detector 2 is used as centralize node + +print("Dectector 3", detector3.elems) # All detectors have been updated +# --8<-- [end:example_3] + +# --8<-- [start:example_4] +detector1 = NewComponent([1, 2, 3]) +detector2 = NewComponent([4, 5]) +detector3 = NewComponent([6]) + +detector1.stack([detector2, detector3]) + +detector2.aggregate() +print("Detector 3", detector3.elems) # It is not longer combine but stack, so it will not work + +detector1.aggregate() +print("Detector 3", detector3.elems) # Now it will work + +# --8<-- [end:example_4] + +# --8<-- [start:example_5] +detector1 = NewComponent([1, 2, 3]) + +binary2 = NewComponent([4, 5]).to_binary() +binary3 = (detector3 := NewComponent([6])).to_binary() +print("Binary of Detector 3", binary3) + +detector1.stack([binary2, binary3]) +output = detector1.aggregate(unstack=True) # unstack = True will free memory + +print("Detector 1", detector1.elems) # Detector 1 has been updated +print("Output binary", output) # Output that we send to other componets + +# We update detector 3 now +print("Detector 3", detector3.elems) # Now detector 3 is not in the share memory so it will not work +detector3 = detector3.from_binary(output) +print("Detector 3", detector3.elems) # Now it will work +# --8<-- [end:example_5] diff --git a/docs/examples/parsers/drain_parser.py b/docs/examples/parsers/drain_parser.py new file mode 100644 index 00000000..cfd949a4 --- /dev/null +++ b/docs/examples/parsers/drain_parser.py @@ -0,0 +1,70 @@ +# flake8: noqa + +# --8<-- [start:example_1] +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" +# --8<-- [end:example_1] + + +# --8<-- [start:example_2] +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "template not found" +# --8<-- [end:example_2] diff --git a/docs/federation.md b/docs/federation.md new file mode 100644 index 00000000..d531b5a4 --- /dev/null +++ b/docs/federation.md @@ -0,0 +1,63 @@ +# Federation + +This section explains how to use the federation setup. For a component to support federation, it must implement the following methods: + +```python +def to_binary(self) -> bytes | None: + """(Federation only) Serialize to bytes for federation operations.""" + +def from_binary(self, binary: bytes) -> object: + """(Federation only) Deserialize from bytes for federation operations.""" + +def aggregate_strategy(self, components: set["FedOperations"]) -> None: + """(Federation only) Define how to aggregate a set of federated components.""" +``` + +There are two main ways to use federation: + +- **Combine first**: This can only be used when all components run locally. The main idea is to simplify the process by allowing components to share memory. +- **Stack later**: A more standard federated approach where the "weights" or state of each component are combined at the end. + +## Example class + +For all the examples below, we will use this code: + +```python +--8<-- "docs/examples/others/federation.py:example_1" +``` + +## Combine first + +The diagram below shows the workflow: + +![combine](img/fed_combine_first.png) + +Example 1: + +```python +--8<-- "docs/examples/others/federation.py:example_2" +``` + +Example 2: + +```python +--8<-- "docs/examples/others/federation.py:example_3" +``` + +## Stack + +The diagram below shows the workflow: + +![stack](img/fed_stack_later.png) + +Example 1: + +```python +--8<-- "docs/examples/others/federation.py:example_4" +``` + +Example 2: + +```python +--8<-- "docs/examples/others/federation.py:example_5" +``` diff --git a/docs/img/fed_combine_first.png b/docs/img/fed_combine_first.png new file mode 100644 index 00000000..4a923e09 Binary files /dev/null and b/docs/img/fed_combine_first.png differ diff --git a/docs/img/fed_stack_later.png b/docs/img/fed_stack_later.png new file mode 100644 index 00000000..42a2edf1 Binary files /dev/null and b/docs/img/fed_stack_later.png differ diff --git a/docs/overall_architecture.md b/docs/overall_architecture.md index 49f8e821..77850f6e 100644 --- a/docs/overall_architecture.md +++ b/docs/overall_architecture.md @@ -70,14 +70,42 @@ class Component(CoreComponent): * Default: the component is just processing data """ + def export_state( + self, path: str | None = None, storage_options: dict[str, Any] | None = None, + ) -> bytes | None: + """Export the current state if persistency class was implemented""" + + def import_state( + self, path: str | bytes, storage_options: dict[str, Any] | None = None + ) -> None: + """Import the current state if persistency class was implemented""" + def process(self, data: BaseSchema | bytes) -> BaseSchema | bytes | None: """Process the data in a stream fashion (Defined in the CoreComponent)""" def get_config(self) -> Dict[str, Any]: - """"Get the configuration of the component (Defined in the CoreComponent)""" + """Get the configuration of the component (Defined in the CoreComponent)""" def update_config(self, new_config: Dict[str, Any]) -> None: - """"Update the configuration of the component (Defined in the CoreComponent)""" + """Update the configuration of the component (Defined in the CoreComponent)""" + + def get_window_size(self) -> int: + """Get window size of the data buffer""" + + def stack(self, other: object | list[object | bytes] | bytes) -> None: + """(Federation only) stack multiple components for federation tasks""" + + def aggregate(self, unstack: bool = False) -> None | bytes: + """(Federation only) aggregate multiple components""" + + def to_binary(self) -> bytes | None: + """(Federation only) fill it to be compatible with federation ops""" + + def from_binary(self, binary: bytes) -> object: + """(Federation only) fill it to be compatible with federation ops""" + + def aggregate_strategy(self, components: set["FedOperations"]) -> None: + """(Federation only) fill it to be compatible with federation ops""" ``` Go back [Index](index.md) diff --git a/docs/parsers.md b/docs/parsers.md index a862c877..2db5fc08 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -108,5 +108,6 @@ def test_my_parser_parse(): - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates. - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure. - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data. +- [Drain parser](parsers/drain_parser.md): Parser inspired by [Drain Publication](https://ieeexplore.ieee.org/document/8029742). Go back to [Index](index.md) diff --git a/docs/parsers/drain_parser.md b/docs/parsers/drain_parser.md new file mode 100644 index 00000000..f20c6afb --- /dev/null +++ b/docs/parsers/drain_parser.md @@ -0,0 +1,47 @@ +# Drain parser + +The parser is derived from the official [Drain publication](https://ieeexplore.ieee.org/document/8029742). + +It also wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. When parsing large numbers of log lines in non-stream (batch) mode, it is recommended to use the performance-oriented implementation. + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [LogSchema](../schemas.md) | Unstructured log | +| **Output** | [ParserSchema](../schemas.md) | Structured log | + +## Configuration + +Drain parser parameters: + +- `method_type` (string): identifier for the parser type (e.g., `"tree_matcher"`). +- `depth` (int): number of token/word levels. +- `max_childs` (int): maximum number of children allowed in the given layer. +- `sim_thres` (float): threshold used for similarity. +- `reset_in_post_train` (bool): if enabled, clears logs from the training buffer once templates are created; otherwise, it keeps them for the next training cycle. +- `auto_config` (bool): indicates whether to run an optional auto-configuration step (not mandatory). + +Example YAML fragment: +```yaml +parsers: + DrainParser: + method_type: drain_parser + auto_config: False + params: + depth: 2 +``` + +## Usage example + +Simple usage (Reset = False): + +```python +--8<-- "docs/examples/parsers/drain_parser.py:example_1" +``` + +Simple usage (Reset = True): + +```python +--8<-- "docs/examples/parsers/drain_parser.py:example_2" +``` + +Go back to [Index](../index.md) diff --git a/mkdocs.yml b/mkdocs.yml index d44b3fbb..85f737c2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Basic concepts: basic_idea.md - Overall architecture: overall_architecture.md - Schemas: schemas.md + - Federation: federation.md - Parsers: parsers.md - Detectors: detectors.md - Alert Aggregation: alert_aggregator.md @@ -22,6 +23,7 @@ nav: - Template Tree Matcher: parsers/template_tree_matcher.md - Json Parser: parsers/json_parser.md - LogBatcher Parser: parsers/logbatcher_parser.md + - Drain Parser: parsers/drain_parser.md - Detectors Methods: - Random Detector: detectors/random_detector.md - New Value: detectors/new_value.md diff --git a/pyproject.toml b/pyproject.toml index 857ae87b..b29abd9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pyyaml>=6.0.3", "regex>=2025.11.3", "numpy>=2.3.2", - "detectmateperformance>=0.1.0", + "detectmateperformance>=0.1.5", "msgpack>=1.0.0", "fsspec>=2024.1.0", "pyarrow>=24.0.0", diff --git a/src/detectmatelibrary/common/_config/__init__.py b/src/detectmatelibrary/common/_config/__init__.py index 12b86205..f7b7d60e 100644 --- a/src/detectmatelibrary/common/_config/__init__.py +++ b/src/detectmatelibrary/common/_config/__init__.py @@ -1,7 +1,14 @@ -from ._compile import ConfigMethods, generate_detector_config +from ._compile import ConfigMethods, generate_detector_config, generate_events_config from ._formats import EventsConfig -__all__ = ["ConfigMethods", "generate_detector_config", "EventsConfig", "BasicConfig"] +__all__ = [ + "ConfigMethods", + "generate_detector_config", + "generate_events_config", + "EventsConfig", + "BasicConfig", + "AutoConfigParams", +] from pydantic import BaseModel, ConfigDict @@ -18,6 +25,25 @@ def random_id(length: int = 10) -> str: return "".join(str(choice(characters)) for _ in range(length)) +class AutoConfigParams(BaseModel): + """Inputs to the auto-configuration (configure) phase. + + Empty here: no component has configure-phase inputs by default. Subclasses + add the fields their own configure phase reads. Kept apart from the + operational `params` block so the phase a setting belongs to is visible in + the YAML, not just in the code that reads it. + + Lives beside `auto_config` on `BasicConfig` rather than on any one + component type: `auto_config` and `Component.configure()` are both declared + at the base, so the block that feeds that phase belongs there too. Empty by + default, and `to_dict` omits it while it stays at its default, so a + component whose configure phase takes no inputs serializes exactly as it + did before this block existed. + """ + + model_config = ConfigDict(extra="forbid") + + class BasicConfig(BaseModel): """Base configuration class with helper methods.""" @@ -27,6 +53,7 @@ class BasicConfig(BaseModel): component_type: str = "default_type" auto_config: bool = False + auto_config_params: AutoConfigParams = AutoConfigParams() def get_config(self) -> Dict[str, Any]: """Return the configuration as a dictionary.""" @@ -69,6 +96,7 @@ def to_dict(self, method_id: str = random_id()) -> Dict[str, Any]: events_data = None instances_data = None persist_data: dict[str, Any] | None = None + auto_params_data: dict[str, Any] | None = None for field_name, field_value in self: # Skip meta fields @@ -82,6 +110,12 @@ def to_dict(self, method_id: str = random_id()) -> Dict[str, Any]: events_data = field_value.to_dict() else: events_data = field_value + # Its own top-level block, and only when it differs from the + # default -- a config that never touches auto-config must serialize + # exactly as it did before this block existed. + elif field_name == "auto_config_params": + if field_value != type(self).model_fields[field_name].default: + auto_params_data = field_value.model_dump() # Handle global instances specially (top-level, not in params) # Serialized as "global" in YAML (Python field is "global_instances") elif field_name == "global_instances" and field_value: @@ -100,6 +134,9 @@ def to_dict(self, method_id: str = random_id()) -> Dict[str, Any]: if params: result["params"] = params + if auto_params_data is not None: + result["auto_config_params"] = auto_params_data + # Add global instances if they exist (serialized as "global" in YAML) if instances_data is not None: result["global"] = instances_data diff --git a/src/detectmatelibrary/common/_config/_compile.py b/src/detectmatelibrary/common/_config/_compile.py index cb99210f..97eed6d2 100644 --- a/src/detectmatelibrary/common/_config/_compile.py +++ b/src/detectmatelibrary/common/_config/_compile.py @@ -64,16 +64,11 @@ def __init__(self, expected_type: str, actual_type: str) -> None: class MissingParamsWarning(UserWarning): def __init__(self) -> None: super().__init__( - "'auto_config = False' and no 'params', 'events', 'global', or 'persist' provided. " - "Is that intended?" + "'auto_config = False' and no 'params', 'auto_config_params', 'events', " + "'global', or 'persist' provided. Is that intended?" ) -class AutoConfigWarning(UserWarning): - def __init__(self) -> None: - super().__init__("'auto_config = True' will overwrite 'events' and 'params'.") - - class ConfigMethods: @staticmethod def get_method( @@ -97,18 +92,18 @@ def check_type(config: Dict[str, Any], method_type: str) -> None: @staticmethod def process(config: Dict[str, Any]) -> Dict[str, Any]: has_params = "params" in config + has_auto_params = "auto_config_params" in config has_events = "events" in config has_instances = "global" in config has_persist = "persist" in config - no_data = not has_params and not has_events and not has_instances and not has_persist + no_data = not ( + has_params or has_auto_params or has_events or has_instances or has_persist + ) if no_data and not config.get("auto_config", False): warnings.warn(MissingParamsWarning()) if has_params: - if config.get("auto_config", False): - warnings.warn(AutoConfigWarning()) - config.update(config["params"]) config.pop("params") @@ -127,6 +122,60 @@ def process(config: Dict[str, Any]) -> Dict[str, Any]: return config +def _build_events_config( + variable_selection: Dict[int | str, List[Union[str, Tuple[str, ...]]]], + detector_name: str, +) -> Dict[int | str, Dict[str, Any]]: + """Map each event_id to its instance dict. + + Shared by the two generators below. + """ + var_pattern = re.compile(r"^var_(\d+)$") + + events_config: Dict[int | str, Dict[str, Any]] = {} + + for event_id, variable_names in variable_selection.items(): + instances: Dict[str, Any] = {} + + # Separate plain strings from tuples + single_vars: List[str] = [] + tuple_vars: List[Tuple[str, ...]] = [] + + for entry in variable_names: + if isinstance(entry, tuple): + tuple_vars.append(entry) + else: + single_vars.append(entry) + + # Plain strings -> single instance keyed by detector_name + if single_vars: + instances[detector_name] = _classify_variables(single_vars, var_pattern) + + # Each tuple -> its own instance, keyed by joined variable names + for combo in tuple_vars: + instance_id = f"{detector_name}_{'_'.join(combo)}" + instances[instance_id] = _classify_variables(combo, var_pattern) + + events_config[event_id] = instances + + return events_config + + +def generate_events_config( + variable_selection: Dict[int | str, List[Union[str, Tuple[str, ...]]]], + detector_name: str, +) -> EventsConfig: + """The `events` block for a variable selection, as the model + set_configuration assigns. + + The configure phase produces exactly this. Everything else on a + detector config is operator input and must survive the phase + untouched, which is why set_configuration writes this instead of + rebuilding the config from generate_detector_config. + """ + return EventsConfig._init(_build_events_config(variable_selection, detector_name)) + + def generate_detector_config( variable_selection: Dict[int | str, List[Union[str, Tuple[str, ...]]]], detector_name: str, @@ -140,6 +189,12 @@ def generate_detector_config( names (strings) and tuples of variable names. Each tuple produces a separate detector instance in the config. + Has no production callers: `set_configuration` writes only `config.events` + via `generate_events_config` (above), never rebuilds the whole config, so + that operator-set `params` and `auto_config_params` survive untouched. This + helper remains for callers that want a complete, standalone config dict + from a variable selection. + Args: variable_selection: Maps event_id to list of variable names or tuples of variable names. Strings are grouped into a single instance. @@ -148,8 +203,15 @@ def generate_detector_config( variables. detector_name: Name of the detector, used as the base instance_id. method_type: Type of detection method (e.g., "new_value_detector"). - **additional_params: Additional parameters for the detector's params - dict (e.g., max_combo_size=3). + **additional_params: Additional parameters for the detector's flat + `params` dict — operational settings read during training and + detection (e.g. `data_use_training=500`). `ConfigMethods.process` + flattens `params` onto the top level and the config classes are + `extra="forbid"`, so a key here must name an actual flat field on + the target config class. Configure-phase-only settings that now + live under `auto_config_params` (e.g. `max_combo_size`) do not + belong here — passing one raises `ValidationError` when the + result is loaded with `.from_dict`. Returns: Dictionary with structure compatible with detector config classes. @@ -163,50 +225,23 @@ def generate_detector_config( method_type="new_value_detector", ) - Tuples of variable names (one instance per tuple):: + Tuples of variable names (one instance per tuple), with an + additional flat operational parameter:: config = generate_detector_config( variable_selection={1: [("username", "src_ip"), ("var_0", "var_1")]}, detector_name="MyDetector", method_type="new_value_combo_detector", - max_combo_size=2, + data_use_training=500, ) """ - var_pattern = re.compile(r"^var_(\d+)$") - - events_config: Dict[int | str, Dict[str, Any]] = {} - - for event_id, variable_names in variable_selection.items(): - instances: Dict[str, Any] = {} - - # Separate plain strings from tuples - single_vars: List[str] = [] - tuple_vars: List[Tuple[str, ...]] = [] - - for entry in variable_names: - if isinstance(entry, tuple): - tuple_vars.append(entry) - else: - single_vars.append(entry) - - # Plain strings -> single instance keyed by detector_name - if single_vars: - instances[detector_name] = _classify_variables(single_vars, var_pattern) - - # Each tuple -> its own instance, keyed by joined variable names - for combo in tuple_vars: - instance_id = f"{detector_name}_{'_'.join(combo)}" - instances[instance_id] = _classify_variables(combo, var_pattern) - - events_config[event_id] = instances - config_dict = { "detectors": { detector_name: { "method_type": method_type, "auto_config": False, "params": additional_params, - "events": events_config + "events": _build_events_config(variable_selection, detector_name), } } } diff --git a/src/detectmatelibrary/common/_core_op/_basic_component.py b/src/detectmatelibrary/common/_core_op/_basic_component.py new file mode 100644 index 00000000..143a8e99 --- /dev/null +++ b/src/detectmatelibrary/common/_core_op/_basic_component.py @@ -0,0 +1,55 @@ +from detectmatelibrary.utils.persistency.component_interfaces import Stoppable +from detectmatelibrary.schemas import BaseSchema + +from detectmatelibrary.common._config import BasicConfig + +from typing import Any, Dict, List + + +class Component: + """Empty methods.""" + def __init__( + self, + name: str, + type_: str = "Core", + config: BasicConfig = BasicConfig(), + ) -> None: + self.name, self.type_, self.config = name, type_, config + self.saver: Stoppable | None = None + + def __repr__(self) -> str: + return f"<{self.type_}> {self.name}: {self.config}" + + def run( + self, input_: List[BaseSchema] | BaseSchema, output_: BaseSchema + ) -> bool: + return False + + def train( + self, input_: List[BaseSchema] | BaseSchema, + ) -> None: + pass + + def configure( + self, input_: List[BaseSchema] | BaseSchema, + ) -> None: + pass + + def set_configuration(self) -> None: + pass + + def post_train(self) -> None: + pass + + def get_config(self) -> Dict[str, Any]: + return self.config.get_config() + + def update_config(self, new_config: Dict[str, Any]) -> None: + self.config.update_config(new_config) + + def __enter__(self) -> "Component": + return self + + def __exit__(self, *_: Any) -> None: + if self.saver is not None: + self.saver.stop() diff --git a/src/detectmatelibrary/common/_core_op/_fed_component.py b/src/detectmatelibrary/common/_core_op/_fed_component.py new file mode 100644 index 00000000..eef4900d --- /dev/null +++ b/src/detectmatelibrary/common/_core_op/_fed_component.py @@ -0,0 +1,108 @@ + +import warnings +from typing import Self, overload + + +class IncompatibleFed(Exception): + def __init__(self) -> None: + super().__init__("Instances are incompatible") + + +class _CompOp: + @staticmethod + def is_compatible(main_inst: object, other_inst: object) -> None: + if not isinstance(other_inst, type(main_inst)): + raise IncompatibleFed() + + @staticmethod + def reset(main_inst: object, attr: str) -> None: + main_inst.__setattr__(attr, {main_inst}) + + @staticmethod + def combine(main_inst: object, attr: str, other_inst: object) -> None: + _CompOp.is_compatible(main_inst, other_inst) + set_main: set[object] = getattr(main_inst, attr) + + set_main.update(getattr(other_inst, attr)) + for elem in set_main: + getattr(elem, attr).update(set_main) + + @staticmethod + def uncombine(main_inst: object, attr: str, other_inst: object) -> None: + _CompOp.is_compatible(main_inst, other_inst) + set_main: set[object] = getattr(main_inst, attr) + + for elem in list(set_main): + if elem == other_inst: + _CompOp.reset(other_inst, attr) + else: + elem._components.remove(other_inst) # type: ignore + + @staticmethod + def stack(main_inst: object, attr: str, list_other_inst: list[object]) -> None: + set_main: set[object] = getattr(main_inst, attr) + for other_inst in list_other_inst: + _CompOp.is_compatible(main_inst, other_inst) + set_main.add(other_inst) + + +class FedOperations: + """Operations related to the federation learning / agregation.""" + __COMPONENT: str = "_components" + + def __init__(self) -> None: + self._components: set["FedOperations"] + _CompOp.reset(self, self.__COMPONENT) + + def __add__(self, other: object) -> Self: + """Add other components to do the aggregation in a combine first + approach.""" + _CompOp.combine(self, attr=self.__COMPONENT, other_inst=other) + + return self + + def __sub__(self, other: object) -> Self: + """Remove other components to do the aggregation in a combine first + approach.""" + _CompOp.uncombine(self, attr=self.__COMPONENT, other_inst=other) + + return self + + @overload + def stack(self, other: bytes | list[bytes]) -> None: + pass + + @overload + def stack(self, other: object | list[object]) -> None: + """Stack other components to do the aggregation in a stack later + approach.""" + pass + + def stack(self, other: object | list[object | bytes] | bytes) -> None: + if not isinstance(other, list): + other = [other] + _CompOp.stack( + self, attr=self.__COMPONENT, list_other_inst=[ + self.from_binary(inst) if isinstance(inst, bytes) else inst for inst in other + ] + ) + + def aggregate(self, unstack: bool = False) -> None | bytes: + self.aggregate_strategy(self._components) + + if unstack: + _CompOp.reset(self, self.__COMPONENT) + + return self.to_binary() + + def to_binary(self) -> bytes | None: + warnings.warn("To binary not implemented, return None") + return None + + def from_binary(self, binary: bytes) -> object: + warnings.warn(f"From binary not implemented, return None for {binary!r}") + return None + + def aggregate_strategy(self, components: set["FedOperations"]) -> None: + """Aggregation strategy use by the component.""" + warnings.warn(f"No strategy found, aggregations does nothing for {components}") diff --git a/src/detectmatelibrary/common/core.py b/src/detectmatelibrary/common/core.py index 7d52d54c..75a3c645 100644 --- a/src/detectmatelibrary/common/core.py +++ b/src/detectmatelibrary/common/core.py @@ -1,5 +1,7 @@ from detectmatelibrary.common._core_op._fit_logic import FitLogicState, StatesL from detectmatelibrary.common._core_op._schema_pipeline import SchemaPipeline +from detectmatelibrary.common._core_op._fed_component import FedOperations +from detectmatelibrary.common._core_op._basic_component import Component from detectmatelibrary.common._core_op._fit_logic import FitLogic from detectmatelibrary.utils.data_buffer import DataBuffer, ArgsBuffer, BufferMode @@ -12,10 +14,9 @@ from detectmatelibrary.tools.logging import logger, setup_logging -from typing import Any, Dict, List +from typing import Any from detectmatelibrary.utils.persistency.component_interfaces import PersistencyOp -from detectmatelibrary.utils.persistency.component_interfaces import Stoppable setup_logging() @@ -43,7 +44,7 @@ def __iter__(self) -> "TrainBuffer": return self -# Core component skeleton structure ################################################ +# Core component ################################################ class CoreConfig(BasicConfig): start_id: int = 10 @@ -52,58 +53,7 @@ class CoreConfig(BasicConfig): use_config_data_as_training: bool = True -class Component: - """Empty methods.""" - def __init__( - self, - name: str, - type_: str = "Core", - config: CoreConfig = CoreConfig(), - ) -> None: - self.name, self.type_, self.config = name, type_, config - self.saver: Stoppable | None = None - - def __repr__(self) -> str: - return f"<{self.type_}> {self.name}: {self.config}" - - def run( - self, input_: List[BaseSchema] | BaseSchema, output_: BaseSchema - ) -> bool: - return False - - def train( - self, input_: List[BaseSchema] | BaseSchema, - ) -> None: - pass - - def configure( - self, input_: List[BaseSchema] | BaseSchema, - ) -> None: - pass - - def set_configuration(self) -> None: - pass - - def post_train(self) -> None: - pass - - def get_config(self) -> Dict[str, Any]: - return self.config.get_config() - - def update_config(self, new_config: Dict[str, Any]) -> None: - self.config.update_config(new_config) - - def __enter__(self) -> "Component": - return self - - def __exit__(self, *_: Any) -> None: - if self.saver is not None: - self.saver.stop() - - -# Core component ################################################ - -class CoreComponent(Component): +class CoreComponent(Component, FedOperations): """Base class for all components in the system.""" def __init__( self, @@ -114,7 +64,9 @@ def __init__( input_schema: type[BaseSchema] = BaseSchema, output_schema: type[BaseSchema] = BaseSchema ) -> None: - super().__init__(name=name, type_=type_, config=config) + Component.__init__(self, name=name, type_=type_, config=config) + FedOperations.__init__(self) + self.config: CoreConfig self.input_schema, self.output_schema = input_schema, output_schema self.data_buffer = DataBuffer(args_buffer) diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py index 1189483e..25a88b58 100644 --- a/src/detectmatelibrary/common/deeplearning_detector.py +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -7,6 +7,7 @@ from detectmatelibrary import schemas from typing import Any +import logging class DeepLearningDetectorConfig(CoreDetectorConfig): @@ -72,8 +73,8 @@ def post_train(self) -> None: if "top_k" in self.stats: self.top_k = int(self.stats["top_k"]) - print(self.model) - print("Top k assigned", self.top_k) + logging.info(self.model) + logging.info(f"Top k assigned {self.top_k}") def detect( self, diff --git a/src/detectmatelibrary/common/detector.py b/src/detectmatelibrary/common/detector.py index 0efc2c99..04735c4e 100644 --- a/src/detectmatelibrary/common/detector.py +++ b/src/detectmatelibrary/common/detector.py @@ -1,4 +1,6 @@ from detectmatelibrary.common._config._formats import EventsConfig, _EventInstance +# Re-exported: subclasses spell it `from detectmatelibrary.common.detector import AutoConfigParams`. +from detectmatelibrary.common._config import AutoConfigParams as AutoConfigParams # noqa: F401 from detectmatelibrary.common.core import CoreComponent, CoreConfig from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode diff --git a/src/detectmatelibrary/common/variable_detector.py b/src/detectmatelibrary/common/variable_detector.py index 6613af48..32c41daf 100644 --- a/src/detectmatelibrary/common/variable_detector.py +++ b/src/detectmatelibrary/common/variable_detector.py @@ -1,6 +1,10 @@ -from detectmatelibrary.common._config._formats import _EventInstance, EventsConfig -from detectmatelibrary.common._config._compile import generate_detector_config, get_configured_variables +from detectmatelibrary.common._config._formats import _EventInstance +from detectmatelibrary.common._config._compile import ( + generate_events_config, + get_configured_variables, +) from detectmatelibrary.common.detector import ( + AutoConfigParams, CoreDetectorConfig, CoreDetector, ) @@ -11,6 +15,9 @@ EventStabilityTracker, SingleStabilityTracker, ) +from detectmatelibrary.utils.persistency.event_data_structures.trackers.stability import ( + ClassificationMethods, +) from detectmatelibrary.utils.persistency.event_persistency import EventPersistency from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.utils.time_format_handler import TimeFormatHandler @@ -18,7 +25,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.tools.logging import logger -from typing import Any, Dict, Literal, Optional, cast +from typing import Any, Dict, Optional, cast from typing_extensions import override @@ -43,20 +50,53 @@ def get_global_variables( return result -class VariableDetectorConfig(CoreDetectorConfig): +def _strip_auto_config_params(detector_config: Dict[str, Any], method_id: str) -> Dict[str, Any]: + """Return a copy of a serialized detector_config with its + auto_config_params block removed. + + detector_config is stashed on a tracker and persisted verbatim by + to_state(). auto_config_params are configure-phase-only inputs -- + the standing constraint is that persisted tracker state never + carries them. Stripped here, at the point the kwargs are built, so + the block never reaches state in the first place. + """ + entry = detector_config.get("detectors", {}).get(method_id, {}) + if "auto_config_params" not in entry: + return detector_config + return { + **detector_config, + "detectors": { + **detector_config["detectors"], + method_id: {k: v for k, v in entry.items() if k != "auto_config_params"}, + }, + } + + +class VariableAutoConfigParams(AutoConfigParams): + """Configure-phase inputs shared by every VariableDetector subclass. + + Read only while `auto_config` is True: stability classification decides + which variables land in the generated `events` block and is never consulted + at detection time. + """ + use_stable_vars: bool = True use_static_vars: bool = True - # Stability segmentation. "count" cuts the classifier's segments at equal - # sample counts (the historical behaviour). "time" cuts them at equal - # durations instead. "both" requires the variable to pass under *both* - # segmentations. The two time-aware modes need a per-record event time, - # named here and read from the record's logFormatVariables. - stability_segmentation: Literal["count", "time", "both"] = "count" + # Which stability classification methods decide STABLE, and how their + # verdicts combine. Four independent methods over two primitives and two + # axes; see ClassificationMethods. The two time-axis methods (`time`, + # `slope_time`) need a per-record event time, named here and read from the + # record's logFormatVariables. + classification: ClassificationMethods = ClassificationMethods() timestamp_variable: str | None = None timestamp_format: str | None = None # None -> TimeFormatHandler auto-detect +class VariableDetectorConfig(CoreDetectorConfig): + auto_config_params: VariableAutoConfigParams = VariableAutoConfigParams() + + class VariableDetector(CoreDetector): """Abstract base for detectors that learn a per-variable model from configured log variables and flag anomalous values at detection time. @@ -79,27 +119,37 @@ def __init__(self, name: str, config: VariableDetectorConfig) -> None: self._warned_bad_timestamp = False self.persistency = EventPersistency( event_data_class=self._event_data_class(), - event_data_kwargs=self._with_segmentation(self._event_data_kwargs()), + # No classification kwargs: the trained trackers are read by + # _check_variable, which looks at unique_set / min-max / charset + # directly and never calls classify(). A classification block + # would only make them collect timestamps nothing reads. + event_data_kwargs=self._event_data_kwargs(), ) # auto config checks individual-variable stability to select features self.auto_conf_persistency = EventPersistency( event_data_class=self._event_data_class(), - event_data_kwargs=self._with_segmentation(self._auto_conf_kwargs()), + event_data_kwargs=self._with_classification_kwargs(self._auto_conf_kwargs()), ) self._register_persistency(self.persistency) - def _with_segmentation(self, kwargs: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: - """Add the segmentation mode to tracker kwargs when it is not the + def _with_classification_kwargs( + self, kwargs: Optional[Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + """Add the classification block to tracker kwargs, unless it is the default. - Done here rather than in _stability_kwargs so every - VariableDetector subclass is covered -- NewValueDetector - overrides neither construction hook and NewValueComboDetector - returns only a converter_function. + Done here rather than in _stability_kwargs so every VariableDetector + subclass is covered -- NewValueDetector overrides neither construction + hook and NewValueComboDetector returns only a converter_function. + + Non-defaults only: forwarding the default block would be noise, and a + block naming a time-axis method would make every variable collect + timestamps it never reads. """ - if self.config.stability_segmentation == "count": + auto = self.config.auto_config_params + if auto.classification == ClassificationMethods(): return kwargs - return {**(kwargs or {}), "segmentation": self.config.stability_segmentation} + return {**(kwargs or {}), "classification": auto.classification.model_dump()} # ---- construction hooks ------------------------------------------------- @@ -118,7 +168,7 @@ def _stability_kwargs(self) -> Dict[str, Any]: name = type(self).__name__ return { "add_value_fn": name, - "detector_config": self.config.to_dict(method_id=name), + "detector_config": _strip_auto_config_params(self.config.to_dict(method_id=name), name), } def _warn_time_fallback_once(self, reason: str) -> None: @@ -131,28 +181,30 @@ def _warn_time_fallback_once(self, reason: str) -> None: return self._warned_bad_timestamp = True logger.warning( - "%s: %s; falling back to count-based stability segmentation.", + "%s: %s; falling back to the index axis for stability classification.", self.name, reason, ) def _timestamp(self, input_: ParserSchema) -> float | None: - """Resolve the record's event time, or None to use count - segmentation.""" - if self.config.stability_segmentation == "count": + """Resolve the record's event time, or None if no enabled + classification method reads the time axis.""" + auto = self.config.auto_config_params + if not auto.classification.needs_timestamps: return None - if not self.config.timestamp_variable: - # Selecting a time-aware mode without naming the field is an operator - # error, not an opt-out -- say so rather than silently no-op. + if not auto.timestamp_variable: + # Selecting a time-axis method without naming the field is an + # operator error, not an opt-out -- say so rather than silently + # no-op. self._warn_time_fallback_once( - f"stability_segmentation is {self.config.stability_segmentation!r} " + "a time-axis classification method is enabled " "but timestamp_variable is not set" ) return None - raw = input_["logFormatVariables"].get(self.config.timestamp_variable) - ts = self._time_handler.parse_timestamp(str(raw or ""), self.config.timestamp_format) + raw = input_["logFormatVariables"].get(auto.timestamp_variable) + ts = self._time_handler.parse_timestamp(str(raw or ""), auto.timestamp_format) if ts == "0": self._warn_time_fallback_once( - f"timestamp_variable {self.config.timestamp_variable!r} is missing or " + f"timestamp_variable {auto.timestamp_variable!r} is missing or " f"unparseable (got {raw!r})" ) return None @@ -195,7 +247,6 @@ def _ingest(self, input_: ParserSchema, variables: Dict[str, Any], event_id: Any event_id=event_id, event_template=input_["template"], named_variables=variables, - timestamp=self._timestamp(input_), ) def detect(self, input_: ParserSchema, output_: DetectorSchema) -> bool: # type: ignore @@ -271,35 +322,27 @@ def set_configuration(self) -> None: variables: Dict[Any, Any] = {} for event_id, tracker in self.auto_conf_persistency.get_events_data().items(): stability_tracker = cast(EventStabilityTracker, tracker) + auto = self.config.auto_config_params stable = ( stability_tracker.get_features_by_classification("STABLE") - if self.config.use_stable_vars + if auto.use_stable_vars else [] ) static = ( stability_tracker.get_features_by_classification("STATIC") - if self.config.use_static_vars + if auto.use_static_vars else [] ) selected = stable + static if selected: variables[event_id] = selected - old_persist = self.config.persist - old_segmentation = self.config.stability_segmentation - old_timestamp_variable = self.config.timestamp_variable - old_timestamp_format = self.config.timestamp_format - config_dict = generate_detector_config( - variable_selection=variables, - detector_name=self.name, - method_type=self.config.method_type, - ) - self.config = type(self.config).from_dict(config_dict, self.name) - self.config.persist = old_persist - self.config.stability_segmentation = old_segmentation - self.config.timestamp_variable = old_timestamp_variable - self.config.timestamp_format = old_timestamp_format - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: + # Write only what the configure phase produced. Rebuilding the config + # from generate_detector_config is what used to drop operator settings: + # it emits four keys, so everything else had to be carried across by + # hand and a forgotten field failed silently. + self.config.events = generate_events_config(variables, self.name) + self.config.auto_config = False + if not self.config.events.events: logger.warning( f"[{self.name}] auto_config=True generated an empty configuration. " "No stable variables were found in configure-phase data. " diff --git a/src/detectmatelibrary/detectors/ecvc_detector.py b/src/detectmatelibrary/detectors/ecvc_detector.py index 5ff6460a..2b92888b 100644 --- a/src/detectmatelibrary/detectors/ecvc_detector.py +++ b/src/detectmatelibrary/detectors/ecvc_detector.py @@ -1,7 +1,14 @@ -from typing import Any, List +from typing import Any, Collection, List from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig +from detectmatelibrary.utils import persistency from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary.utils.sequence_encoding import ( + build_count_vec, + decode_count_vec, + encode_count_vec, + warn_on_window_size_mismatch, +) from detectmatelibrary import schemas from math import ceil @@ -9,18 +16,7 @@ class ECVCOp: - @staticmethod - def build_count_vec(input_: List[schemas.ParserSchema]) -> tuple[int, ...]: - sequence, n = [0], 0 - for in_ in input_: - event = in_["EventID"] - if n < event: - for _ in range(n, event): - sequence.append(0) - n = event - sequence[event] += 1 - - return tuple(sequence) + build_count_vec = staticmethod(build_count_vec) @staticmethod def build_one_vec(input_: List[schemas.ParserSchema], n: int) -> np.ndarray: @@ -33,7 +29,7 @@ def build_one_vec(input_: List[schemas.ParserSchema], n: int) -> np.ndarray: return arr @staticmethod - def init_count_matrix(seqs: set[tuple[int, ...]]) -> np.ndarray: + def init_count_matrix(seqs: Collection[tuple[int, ...]]) -> np.ndarray: m, n = len(seqs), max([len(s) for s in seqs]) matrix = np.zeros((m, n)) @@ -91,24 +87,63 @@ def __init__( config=config, buffer_size=config.window_size ) - self.train_seqs: set[tuple[int, ...]] = set() self.count_vecs: np.ndarray | None = None self.threshold: float = 0 + # ponytail: only events_seen is used here — count vectors carry no + # variables. EventPersistency still requires an event_data_class. + self.persistency = persistency.EventPersistency( + event_data_class=persistency.EventStabilityTracker, + ) + self._register_persistency(self.persistency) # restores state when auto_load + warn_on_window_size_mismatch(self.name, self.persistency, self.config.window_size) + self._derive() # no-op unless auto_load restored count vectors + + def import_state( + self, path: str | bytes, storage_options: dict[str, Any] | None = None + ) -> None: + """Load state, then rebuild the matrix and threshold from it. + + Unlike `auto_load`, this runs after construction, so the derivation in + `__init__` has already run against an empty store and has to be redone. + """ + super().import_state(path, storage_options) + warn_on_window_size_mismatch(self.name, self.persistency, self.config.window_size) + self._derive() def train(self, input_: List[schemas.ParserSchema]) -> None: # type: ignore - self.train_seqs.add(ECVCOp.build_count_vec(input_)) + self.persistency.ingest_event( + event_id=encode_count_vec(self.config.window_size, ECVCOp.build_count_vec(input_)), + event_template=input_[-1]["template"], + ) - def post_train(self) -> None: - train_idx = ceil(len(self.train_seqs) * (1 - self.config.validation_per)) + def _derive(self) -> None: + """Build the count vector matrix and threshold from the learned + vectors. + + The vectors are sorted first: restored keys are strings, whose set + iteration order is hash-randomized per process, and the seeded shuffle + below splits train from validation by that order. Sorting makes a + restored model identical to a freshly trained one. + """ + seqs = sorted( + decode_count_vec(str(encoded))[1] + for encoded in self.persistency.get_events_seen() + ) + if not seqs: + return + + train_idx = ceil(len(seqs) * (1 - self.config.validation_per)) np.random.seed(self.config.seed) - matrix = ECVCOp.init_count_matrix(self.train_seqs)[np.random.permutation(len(self.train_seqs))] + matrix = ECVCOp.init_count_matrix(seqs)[np.random.permutation(len(seqs))] self.count_vecs, val = matrix[:train_idx], matrix[train_idx:] if len(val) > 0: self.threshold = ECVCOp.threshold_cal( y_s=val, matrix=self.count_vecs, method=self.config.threshold_method ) - self.train_seqs = set() + + def post_train(self) -> None: + self._derive() def detect( self, input_: List[schemas.ParserSchema], output_: schemas.DetectorSchema, # type: ignore diff --git a/src/detectmatelibrary/detectors/event_sequence_detector.py b/src/detectmatelibrary/detectors/event_sequence_detector.py index 7da1f1e0..0e09af66 100644 --- a/src/detectmatelibrary/detectors/event_sequence_detector.py +++ b/src/detectmatelibrary/detectors/event_sequence_detector.py @@ -1,50 +1,50 @@ """Detect EventID sequences that were not observed during training.""" from collections import deque -from typing import Any, Sequence +from typing import Any from pydantic import Field, model_validator -from detectmatelibrary.common._config._compile import generate_detector_config -from detectmatelibrary.common.detector import CoreDetectorConfig, CoreDetector +from detectmatelibrary.common._config._compile import generate_events_config +from detectmatelibrary.common.detector import AutoConfigParams, CoreDetectorConfig, CoreDetector from detectmatelibrary.tools.logging import logger from detectmatelibrary.utils import persistency from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary.utils.sequence_encoding import decode_sequence, encode_sequence from detectmatelibrary.schemas import ParserSchema, DetectorSchema -_SEQUENCE_SEPARATOR = "\x1f" +class SequenceAutoConfigParams(AutoConfigParams): + """Configure-phase inputs: the candidate window lengths to try. -def _encode_sequence(sequence: Sequence[int]) -> str: - return _SEQUENCE_SEPARATOR.join(str(event_id) for event_id in sequence) + @param min_window_size shortest window length tried during the + auto-configuration phase. Only used while `fixed_window_size` is None. + @param max_window_size longest window length tried during the + auto-configuration phase. The longest length whose sequences are + classified STABLE or STATIC wins. + """ + min_window_size: int = Field(default=2, ge=1) + max_window_size: int = Field(default=10, ge=1) -def _decode_sequence(encoded: str) -> tuple[int, ...]: - return tuple(int(event_id) for event_id in encoded.split(_SEQUENCE_SEPARATOR)) + @model_validator(mode="after") + def _validate_window_range(self) -> "SequenceAutoConfigParams": + if self.max_window_size < self.min_window_size: + raise ValueError("max_window_size must be >= min_window_size") + return self class EventSequenceDetectorConfig(CoreDetectorConfig): """ @param fixed_window_size length of the sliding EventID window. A window whose exact EventID sequence was not seen during training is reported as an anomaly. When - set it overrides `min_window_size`/`max_window_size` and skips + set it overrides the `auto_config_params` window range and skips auto-configuration; auto-configuration writes its own choice here. While it is None the detector is unconfigured and neither trains nor alerts. - @param min_window_size shortest window length tried during the auto-configuration - phase. Only used while `fixed_window_size` is None. - @param max_window_size longest window length tried during the auto-configuration - phase. The longest length whose sequences are classified STABLE or STATIC wins. """ method_type: str = "event_sequence_detector" - min_window_size: int = Field(default=2, ge=1) - max_window_size: int = Field(default=10, ge=1) fixed_window_size: int | None = Field(default=None, ge=1) - - @model_validator(mode="after") - def _validate_window_range(self) -> "EventSequenceDetectorConfig": - if self.max_window_size < self.min_window_size: - raise ValueError("max_window_size must be >= min_window_size") - return self + auto_config_params: SequenceAutoConfigParams = SequenceAutoConfigParams() class EventSequenceDetector(CoreDetector): @@ -102,7 +102,7 @@ def _adopt_restored_length(self) -> int | None: restored = self.persistency.get_events_seen() if not restored: return None - length = len(_decode_sequence(str(next(iter(restored))))) + length = len(decode_sequence(str(next(iter(restored))))) if length != self.config.fixed_window_size: logger.warning( f"[{self.name}] restored state holds sequences of length {length}, but " @@ -135,7 +135,7 @@ def train(self, input_: ParserSchema) -> None: # type: ignore if len(self._train_window) < length: return self.persistency.ingest_event( - event_id=_encode_sequence(self._train_window), + event_id=encode_sequence(self._train_window), event_template=input_["template"] ) @@ -152,7 +152,7 @@ def detect(self, input_: ParserSchema, output_: DetectorSchema) -> bool: # type if len(self._detect_window) < length: return False - if _encode_sequence(self._detect_window) in self.persistency.get_events_seen(): + if encode_sequence(self._detect_window) in self.persistency.get_events_seen(): return False sequence = tuple(self._detect_window) @@ -174,7 +174,8 @@ def configure(self, input_: ParserSchema) -> None: # type: ignore """ if self.config.fixed_window_size is not None: return - for length in range(self.config.min_window_size, self.config.max_window_size + 1): + auto = self.config.auto_config_params + for length in range(auto.min_window_size, auto.max_window_size + 1): window = self._configure_windows.setdefault(length, deque(maxlen=length)) window.append(input_["EventID"]) if len(window) == length: @@ -216,22 +217,15 @@ def set_configuration(self) -> None: stable.append(int(length)) if not stable: + auto = self.config.auto_config_params logger.warning( f"[{self.name}] auto_config=True found no stable window size in " - f"[{self.config.min_window_size}..{self.config.max_window_size}]. " + f"[{auto.min_window_size}..{auto.max_window_size}]. " "Generating an empty configuration — no instance of this detector is " "created and it will neither train nor alert." ) - old_persist = self.config.persist - self.config = EventSequenceDetectorConfig.from_dict( - generate_detector_config( - variable_selection={}, - detector_name=self.name, - method_type=self.config.method_type, - ), - self.name, - ) - self.config.persist = old_persist + self.config.events = generate_events_config({}, self.name) + self.config.auto_config = False self._release_configure_state() return @@ -240,7 +234,9 @@ def set_configuration(self) -> None: f"[{self.name}] auto_config selected fixed_window_size={chosen} " f"from stable candidates {sorted(stable)}." ) + self.config.events = generate_events_config({}, self.name) self._set_window_length(chosen) + self.config.auto_config = False self._release_configure_state() def _release_configure_state(self) -> None: @@ -259,6 +255,6 @@ def reset_window(self) -> None: def get_known_sequences(self) -> set[tuple[int, ...]]: """Return the EventID sequences learned during training.""" return { - _decode_sequence(str(encoded)) + decode_sequence(str(encoded)) for encoded in self.persistency.get_events_seen() } diff --git a/src/detectmatelibrary/detectors/new_event_detector.py b/src/detectmatelibrary/detectors/new_event_detector.py index 9d9a5826..133d6213 100644 --- a/src/detectmatelibrary/detectors/new_event_detector.py +++ b/src/detectmatelibrary/detectors/new_event_detector.py @@ -1,4 +1,4 @@ -from detectmatelibrary.common._config._compile import generate_detector_config +from detectmatelibrary.common._config._compile import generate_events_config from detectmatelibrary.common.detector import CoreDetectorConfig, CoreDetector from detectmatelibrary.common.variable_detector import get_global_variables from detectmatelibrary.utils import persistency @@ -88,12 +88,7 @@ def configure(self, input_: ParserSchema) -> None: # type: ignore ) def set_configuration(self) -> None: - old_persist = self.config.persist - config_dict = generate_detector_config( - variable_selection={}, - detector_name=self.name, - method_type=self.config.method_type - ) - # Update the config object from the dictionary instead of replacing it - self.config = NewEventDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist + # This detector keys on EventIDs only -- it selects no variables, so + # the configure phase produces an empty events block. + self.config.events = generate_events_config({}, self.name) + self.config.auto_config = False diff --git a/src/detectmatelibrary/detectors/new_value_combo_detector.py b/src/detectmatelibrary/detectors/new_value_combo_detector.py index 7c5f3ed9..7bdd372b 100644 --- a/src/detectmatelibrary/detectors/new_value_combo_detector.py +++ b/src/detectmatelibrary/detectors/new_value_combo_detector.py @@ -1,6 +1,9 @@ -from detectmatelibrary.common._config import generate_detector_config -from detectmatelibrary.common._config._formats import EventsConfig -from detectmatelibrary.common.variable_detector import VariableDetector, VariableDetectorConfig +from detectmatelibrary.common._config import generate_events_config +from detectmatelibrary.common.variable_detector import ( + VariableDetector, + VariableDetectorConfig, + VariableAutoConfigParams, +) from detectmatelibrary.common._config._compile import get_configured_variables from detectmatelibrary.utils import persistency @@ -39,11 +42,19 @@ def get_all_possible_combos( return combo_dict +class ComboAutoConfigParams(VariableAutoConfigParams): + # Combo-detector default, unchanged from the flat field it replaces. + use_static_vars: bool = False + # Longest variable combination the configure phase will consider. Read only + # while auto_config is True: detection reads the combos the phase wrote into + # `events`, never this. + max_combo_size: int = 3 + + class NewValueComboDetectorConfig(VariableDetectorConfig): method_type: str = "new_value_combo_detector" - max_combo_size: int = 3 - use_static_vars: bool = False + auto_config_params: ComboAutoConfigParams = ComboAutoConfigParams() class NewValueComboDetector(VariableDetector): @@ -59,7 +70,7 @@ def __init__( # second-pass persistency to learn stability of variable combinations self.auto_conf_persistency_combos = persistency.EventPersistency( event_data_class=persistency.EventStabilityTracker, - event_data_kwargs=self._with_segmentation( + event_data_kwargs=self._with_classification_kwargs( {"converter_function": get_all_possible_combos} ), ) @@ -102,24 +113,8 @@ def set_configuration(self, max_combo_size: int | None = None) -> None: 3. Re-ingest all events to learn the stability of those combos (testing every possible combo up front would explode combinatorially). """ - old_persist = self.config.persist - segmentation_fields = { - "stability_segmentation": self.config.stability_segmentation, - "timestamp_variable": self.config.timestamp_variable, - "timestamp_format": self.config.timestamp_format, - } - - def restore_segmentation_fields() -> None: - """Carry the segmentation settings across a config reassignment. - - generate_detector_config only emits method_type / auto_config / - params / events, so every ``from_dict`` below resets these to their - defaults. The re-ingest loop calls ``_timestamp()`` under the pass-1 - config, so restoring only at the end would leave the combo trackers - timestamp-less. - """ - for field, value in segmentation_fields.items(): - setattr(self.config, field, value) + if max_combo_size is not None: + self.config.auto_config_params.max_combo_size = max_combo_size # pass 1: stable individual variables -> combos variable_combos = {} @@ -127,14 +122,7 @@ def restore_segmentation_fields() -> None: stable_vars = tracker.get_features_by_classification("STABLE") # type: ignore if len(stable_vars) > 1: variable_combos[event_id] = stable_vars - config_dict = generate_detector_config( - variable_selection=variable_combos, - detector_name=self.name, - method_type=self.config.method_type, - max_combo_size=max_combo_size or self.config.max_combo_size, - ) - self.config = NewValueComboDetectorConfig.from_dict(config_dict, self.name) - restore_segmentation_fields() + self.config.events = generate_events_config(variable_combos, self.name) # re-ingest all inputs to learn combos under the new configuration for input_ in self.inputs: @@ -148,31 +136,24 @@ def restore_segmentation_fields() -> None: # pass 2: stable/static combos -> final config combo_selection = {} + auto = self.config.auto_config_params for event_id, tracker in self.auto_conf_persistency_combos.get_events_data().items(): stable_combos = ( tracker.get_features_by_classification("STABLE") # type: ignore - if self.config.use_stable_vars + if auto.use_stable_vars else [] ) static_combos = ( tracker.get_features_by_classification("STATIC") # type: ignore - if self.config.use_static_vars + if auto.use_static_vars else [] ) combos = stable_combos + static_combos if combos: combo_selection[event_id] = combos - config_dict = generate_detector_config( - variable_selection=combo_selection, - detector_name=self.name, - method_type=self.config.method_type, - max_combo_size=max_combo_size or self.config.max_combo_size, - ) - self.config = NewValueComboDetectorConfig.from_dict(config_dict, self.name) - self.config.persist = old_persist - restore_segmentation_fields() - events = self.config.events - if isinstance(events, EventsConfig) and not events.events: + self.config.events = generate_events_config(combo_selection, self.name) + self.config.auto_config = False + if not self.config.events.events: logger.warning( f"[{self.name}] auto_config=True generated an empty configuration. " "No stable variable combinations were found in configure-phase data. " diff --git a/src/detectmatelibrary/detectors/scvs_detector.py b/src/detectmatelibrary/detectors/scvs_detector.py index a8223de7..e7289be7 100644 --- a/src/detectmatelibrary/detectors/scvs_detector.py +++ b/src/detectmatelibrary/detectors/scvs_detector.py @@ -1,23 +1,17 @@ from typing import Any, List from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig +from detectmatelibrary.utils import persistency from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary.utils.sequence_encoding import ( + build_count_vec, + decode_count_vec, + encode_count_vec, + warn_on_window_size_mismatch, +) from detectmatelibrary import schemas -def build_count_vec(input_: List[schemas.ParserSchema]) -> tuple[int, ...]: - sequence, n = [0], 0 - for in_ in input_: - event = in_["EventID"] - if n < event: - for _ in range(n, event): - sequence.append(0) - n = event - sequence[event] += 1 - - return tuple(sequence) - - class SCVSDetectorConfig(CoreDetectorConfig): method_type: str = "scvs_detector" window_size: int = 10 @@ -40,18 +34,46 @@ def __init__( config=config, buffer_size=config.window_size ) - self.train_seqs: set[tuple[int, ...]] = set() + # ponytail: only events_seen is used here — count vectors carry no + # variables. EventPersistency still requires an event_data_class. + self.persistency = persistency.EventPersistency( + event_data_class=persistency.EventStabilityTracker, + ) + self._register_persistency(self.persistency) # restores state when auto_load + warn_on_window_size_mismatch(self.name, self.persistency, self.config.window_size) + + def import_state( + self, path: str | bytes, storage_options: dict[str, Any] | None = None + ) -> None: + """Load state, then check it was trained at the configured window size. + + Unlike `auto_load`, this runs after construction, so the check in + `__init__` has already passed and has to be redone here. + """ + super().import_state(path, storage_options) + warn_on_window_size_mismatch(self.name, self.persistency, self.config.window_size) def train(self, input_: List[schemas.ParserSchema]) -> None: # type: ignore - self.train_seqs.add(build_count_vec(input_)) + self.persistency.ingest_event( + event_id=encode_count_vec(self.config.window_size, build_count_vec(input_)), + event_template=input_[-1]["template"], + ) def detect( self, input_: List[schemas.ParserSchema], output_: schemas.DetectorSchema, # type: ignore ) -> bool: - if build_count_vec(input_) not in self.train_seqs: + key = encode_count_vec(self.config.window_size, build_count_vec(input_)) + if key not in self.persistency.get_events_seen(): output_["score"] = 1. output_["description"] = "Count vector not found" return True return False + + def get_known_count_vecs(self) -> set[tuple[int, ...]]: + """Return the count vectors learned during training.""" + return { + decode_count_vec(str(encoded))[1] + for encoded in self.persistency.get_events_seen() + } diff --git a/src/detectmatelibrary/metadata.py b/src/detectmatelibrary/metadata.py index bc943ec2..4f566b8b 100644 --- a/src/detectmatelibrary/metadata.py +++ b/src/detectmatelibrary/metadata.py @@ -7,6 +7,6 @@ __website__ = "https://aecid.ait.ac.at" __license__ = "EUPL-1.2" __status__ = "Development" -__version__ = "0.5.2" +__version__ = "0.5.3" __all__ = ['__authors__', '__contact__', '__copyright__', '__date__', '__deprecated__', '__website__', '__license__', '__status__', '__version__'] diff --git a/src/detectmatelibrary/parsers/drain.py b/src/detectmatelibrary/parsers/drain.py index e69de29b..abed9424 100644 --- a/src/detectmatelibrary/parsers/drain.py +++ b/src/detectmatelibrary/parsers/drain.py @@ -0,0 +1,113 @@ +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig +from detectmatelibrary import schemas + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.drain import Drain + +from detectmatelibrary.utils.finetune import Combinations + +from typing import Any + + +class DrainConfig(CoreParserConfig): + method_type: str = "drain_parser" + + depth: int = 2 + max_childs: int = 10 + sim_thres: float = 0.2 + + reset_in_post_train: bool = False + + Finetune: list[list[str | list[Any]]] = [ + ["depth", [1, 2, 3, 4]], + ["max_childs", [10, 40]], + ["sim_thres", [0.2, 0.4, 0.6, 0.8]] + ] + + +def _init_drain(config: DrainConfig) -> Drain: + return Drain( + depth=config.depth, max_child=config.max_childs, sim=config.sim_thres, + ) + + +def _found_ratio(logs: list[str], tree_matcher: TreeMatcher) -> float: + results = tree_matcher.match_batch(logs).get_all_templates() + + score = 0.0 + for template in results: + if "template not found" == template: + score += 1. + + return score / len(results) + + +def _get_best_config(logs: list[str], config: DrainConfig) -> DrainConfig: + + found_ratio: list[float] = [] + length: list[int] = [] + + for config in (comb := Combinations(config))(): # type: ignore + drain = _init_drain(config) + for input_ in logs: + drain.add(input_) + tree_matcher = drain.generate() + + found_ratio.append(_found_ratio(logs, tree_matcher)) + length.append(len(tree_matcher)) + + n = max(length) + for le, sc in zip(length, found_ratio): + comb.add_value((float(le) / n) + sc) + + new_config: DrainConfig = comb.get_best() # type: ignore + return new_config + + +class DrainParser(CoreParser): + def __init__( + self, + name: str = "DrainParser", + config: DrainConfig | dict[str, Any] = DrainConfig() + ) -> None: + + if isinstance(config, dict): + config = DrainConfig.from_dict(config, name) + super().__init__(name=name, config=config) + + self.config: DrainConfig + self.drain_gen = _init_drain(config=config) + self.tree_match: TreeMatcher | None = None + + self.config_buffer: list[str] = [] + + def configure(self, input_: schemas.LogSchema) -> None: # type: ignore + self.config_buffer.append(input_["log"]) + + def set_configuration(self) -> None: + self.config = _get_best_config(self.config_buffer, config=self.config) + self.config_buffer = [] + + def train(self, input_: schemas.LogSchema) -> None: # type: ignore + self.drain_gen.add(input_["log"]) + + def post_train(self) -> None: + self.tree_match = self.drain_gen.generate() + if self.config.reset_in_post_train: + self.drain_gen.reset() + + def parse( + self, + input_: schemas.LogSchema, + output_: schemas.ParserSchema + ) -> None: + + if self.tree_match is None: + output_["EventID"] = -1 + output_["template"] = "templates not yet generated" + else: + parsed = self.tree_match.match_log(input_["log"], get_var=True)[0] + + output_["EventID"] = parsed["EventID"] + output_["variables"].extend(parsed["ParamList"]) + output_["template"] = parsed["Template"] diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/__init__.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/__init__.py index 6b7e531c..d4b7d6b4 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/__init__.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/__init__.py @@ -11,7 +11,8 @@ StabilityClassifier, SingleStabilityTracker, MultiStabilityTracker, - EventStabilityTracker + EventStabilityTracker, + ClassificationMethods, ) from .base import ( EventTracker, @@ -29,4 +30,5 @@ "SingleStabilityTracker", "MultiStabilityTracker", "EventStabilityTracker", + "ClassificationMethods", ] diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/base/event_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/base/event_tracker.py index 7e12103a..55496c2f 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/base/event_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/base/event_tracker.py @@ -72,7 +72,7 @@ def load(cls, data: bytes, **kwargs: Any) -> "EventTracker": ``multi_tracker_type`` recorded in the snapshot. For any subclass, ``cls(**kwargs)`` is called instead, which lets subclasses with closure-based factories (e.g. ``EventStabilityTracker``'s - ``segmentation``) rebuild their factory so it survives load. + ``classification``) rebuild their factory so it survives load. Contract for subclasses: ``__init__`` must accept the kwargs forwarded to ``load()`` and must not require additional positional arguments. diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/__init__.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/__init__.py index 32ead5dd..33a5e10c 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/__init__.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/__init__.py @@ -1,9 +1,11 @@ from .stability_tracker import SingleStabilityTracker, MultiStabilityTracker, EventStabilityTracker from .stability_classifier import StabilityClassifier +from .classification_methods import ClassificationMethods __all__ = [ "EventStabilityTracker", "MultiStabilityTracker", "SingleStabilityTracker", "StabilityClassifier", + "ClassificationMethods", ] diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/classification_methods.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/classification_methods.py new file mode 100644 index 00000000..89a0d826 --- /dev/null +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/classification_methods.py @@ -0,0 +1,62 @@ +"""Which stability classification methods run, and how they combine.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, model_validator + +METHOD_NAMES = ("index", "time", "slope_index", "slope_time") + + +class ClassificationMethods(BaseModel): + """Selection of stability classification methods plus the decision rule. + + Four independent methods, two primitives over two axes:: + + index segment-mean thresholds equal-count boundaries + time segment-mean thresholds equal-duration boundaries + slope_index change centroid index positions + slope_time change centroid normalized timestamps + + Any subset may be enabled and any one may stand alone. The default -- + ``index`` alone under ``consensus`` -- is the historical behaviour. + + ``slope_threshold`` is shared by both slope methods: they are the same + quantity measured on two axes and land on the same [-0.5, +0.5] scale, + so one number keeps them comparable. + """ + + model_config = ConfigDict(extra="forbid") + + index: bool = True + time: bool = False + slope_index: bool = False + slope_time: bool = False + slope_threshold: float = -0.05 + decision: Literal["consensus", "majority"] = "consensus" + + @model_validator(mode="after") + def _at_least_one_method(self) -> "ClassificationMethods": + if not self.enabled: + raise ValueError( + "at least one classification method must be enabled " + f"({', '.join(METHOD_NAMES)}). With none enabled, every variable " + "that is not INSUFFICIENT_DATA, STATIC or RANDOM would be " + "classified STABLE by default -- those three are decided before " + "any method is consulted." + ) + return self + + @property + def enabled(self) -> tuple[str, ...]: + """Enabled method names, in the order they appear in the config + block.""" + return tuple(name for name in METHOD_NAMES if getattr(self, name)) + + @property + def needs_timestamps(self) -> bool: + """Whether any enabled method reads the time axis. + + The tracker gates timestamp collection on this: with only index-axis + methods enabled, recording stamps would cost memory nothing reads. + """ + return self.time or self.slope_time diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_classifier.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_classifier.py index c629547c..c189f2f2 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_classifier.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_classifier.py @@ -1,67 +1,91 @@ """Classifier for stability based on segment means.""" -from typing import List +from typing import Dict, List import numpy as np from detectmatelibrary.utils.persistency.rle_list import RLEList +from .classification_methods import ClassificationMethods + + +def _timestamps_usable(timestamps: List[float] | None, total_len: int) -> bool: + """Whether these timestamps can carry a time-axis computation. + + Shared by ``_segment_boundaries`` (the ``time`` method) and ``slope()`` + (the ``slope_time`` method), so both fall back to the index axis under + exactly the same conditions. ``_slope`` still degrades further on its + own: even when this predicate says yes, it drops to the index axis when + the first observation's offset leaves no usable span (see its + ``u_first >= 1.0`` guard) -- a case ``_segment_boundaries`` has no + equivalent for. + + ``np.searchsorted`` and the centroid both require sorted input. Merged + sources or concurrent writers can deliver stamps out of order, which + would silently produce wrong boundaries rather than an error. O(N), the + same cost as the isfinite scan. + """ + try: + return ( + timestamps is not None + and len(timestamps) == total_len + and total_len > 0 + and bool(np.all(np.isfinite(timestamps))) + and bool(np.all(np.diff(timestamps) >= 0)) + and timestamps[-1] > timestamps[0] + ) + except TypeError: + # e.g. a None entry: not comparable/convertible -> index axis + return False class StabilityClassifier: """Classifier for stability based on segment means.""" - def __init__(self, segment_thresholds: List[float], min_samples: int = 10): + def __init__( + self, + segment_thresholds: List[float], + min_samples: int = 10, + classification: ClassificationMethods | None = None, + ): self.segment_threshs = segment_thresholds self.min_samples = min_samples + self.classification = classification or ClassificationMethods() # for RLELists self.segment_sums = [0.0] * len(segment_thresholds) self.segment_counts = [0] * len(segment_thresholds) self.n_segments = len(self.segment_threshs) # for lists self.segment_means: List[float] = [] + # Transient, rebuilt by every verdicts() call: one human-readable line + # per enabled method, which is what the tracker's reason string is + # assembled from. Never persisted -- it is derived from the series. + self.last_details: Dict[str, str] = {} def _segment_boundaries(self, total_len: int, timestamps: List[float] | None = None) -> List[int]: """Index boundaries of n_segments segments over total_len items. - Equal-count by default. When timestamps are given (one per item, - non-decreasing, non-zero span), boundaries are equal-DURATION - cuts of the observed time span, mapped back to indices. Falls - back to equal-count on missing / mismatched / non-finite / out- - of-order timestamps and on zero span. A duration cut that leaves - a segment empty is kept as-is: nothing observed in that window - means no changes in it, which ``is_stable`` scores as a mean of - 0.0. + Equal-index by default: each segment holds the same number of + observations. When timestamps are given (one per item, non-decreasing, + non-zero span), boundaries are equal-DURATION cuts of the observed time + span, mapped back to indices. Falls back to equal-index whenever + ``_timestamps_usable`` says no. A duration cut that leaves a segment + empty is kept as-is: nothing observed in that window means no changes + in it, which ``is_stable`` scores as a mean of 0.0. """ segment_size = total_len / self.n_segments - count_boundaries = [int(i * segment_size) for i in range(self.n_segments + 1)] - count_boundaries[-1] = total_len - try: - use_time = ( - timestamps is not None - and len(timestamps) == total_len - and total_len > 0 - and bool(np.all(np.isfinite(timestamps))) - # np.searchsorted below requires sorted input. Merged sources or - # concurrent writers can deliver stamps out of order, which would - # silently produce wrong boundaries rather than an error. O(N), - # same cost as the isfinite scan above. - and bool(np.all(np.diff(timestamps) >= 0)) - and timestamps[-1] > timestamps[0] - ) - except TypeError: - # e.g. a None entry: not comparable/convertible -> equal-count - use_time = False - if use_time and timestamps is not None: # 2nd clause narrows for mypy - t_first, t_last = timestamps[0], timestamps[-1] - cuts = [ - t_first + k * (t_last - t_first) / self.n_segments - for k in range(self.n_segments + 1) - ] - boundaries = [int(np.searchsorted(timestamps, t, side="left")) for t in cuts] - boundaries[0] = 0 - boundaries[-1] = total_len - return boundaries - return count_boundaries + index_boundaries = [int(i * segment_size) for i in range(self.n_segments + 1)] + index_boundaries[-1] = total_len + if timestamps is None or not _timestamps_usable(timestamps, total_len): + return index_boundaries + t_first, t_last = timestamps[0], timestamps[-1] + cuts = [ + t_first + k * (t_last - t_first) / self.n_segments + for k in range(self.n_segments + 1) + ] + boundaries = [int(np.searchsorted(timestamps, t, side="left")) for t in cuts] + boundaries[0] = 0 + boundaries[-1] = total_len + return boundaries - def is_stable( + def _segment_verdict( self, change_series: RLEList[bool] | List[bool], timestamps: List[float] | None = None, @@ -75,9 +99,7 @@ def is_stable( the conditions under which time mode falls back to count mode. A segment with no observations in it scores a mean of 0.0 -- no - occurrences means no changes. Equal-duration cuts of a bursty - series leave such segments routinely; pair ``time`` with - ``count`` (segmentation ``both``) if that leniency matters. + occurrences means no changes. """ total_len = len(change_series) if total_len == 0: @@ -123,6 +145,198 @@ def is_stable( ] return all([not q >= thresh for q, thresh in zip(self.segment_means, self.segment_threshs)]) + def verdicts( + self, + change_series: RLEList[bool] | List[bool], + timestamps: List[float] | None = None, + ) -> Dict[str, bool]: + """Per-method stability verdicts, enabled methods only. + + Keys are the method names as they appear in the config block -- + "index", "time", "slope_index", "slope_time" -- in block order, so a + reason string built by iterating this dict reads the same way every + time. + + Every enabled method runs; there is no short-circuit. A combined + verdict is only debuggable if the note carries all of the evidence, + and ``last_details`` is populated here for exactly that. + + ``timestamps`` is threaded through only to the two time-axis methods, + ``time`` and ``slope_time``; ``index`` and ``slope_index`` never see + it. A caller who supplies ``timestamps`` without enabling either + time-axis method gets index-axis results with no error and no + warning. + """ + self.last_details = {} + self.segment_means = [] + out: Dict[str, bool] = {} + empty = len(change_series) == 0 + for name in self.classification.enabled: + if name in ("index", "time"): + stamps = timestamps if name == "time" else None + stable = self._segment_verdict(change_series, timestamps=stamps) + self.last_details[name] = ( + f"{name}: means {self.segment_means} " + f"{'below' if stable else 'exceed'} thresholds " + f"{self.segment_threshs} -> {'STABLE' if stable else 'UNSTABLE'}" + ) + else: + stamps = timestamps if name == "slope_time" else None + if empty: + # Matches the segment methods: an empty series has nothing + # that could have changed, so nothing failed. + stable, k, axis = True, 0.0, "index" + else: + k, axis = self._slope(change_series, stamps) + stable = k <= self.classification.slope_threshold + self.last_details[name] = ( + f"{name}: centroid {k:+.3f} ({axis} axis) " + f"{'at or below' if stable else 'above'} slope_threshold " + f"{self.classification.slope_threshold} -> " + f"{'STABLE' if stable else 'UNSTABLE'}" + ) + out[name] = stable + return out + + def decide(self, verdicts: Dict[str, bool]) -> bool: + """Combine per-method verdicts under the configured decision rule. + + ``consensus`` requires every method; ``majority`` requires strictly + more than half, so a tie resolves to UNSTABLE. The two agree at one + and two enabled methods and diverge at three and four. + """ + if not verdicts: + # Unreachable via the config: ClassificationMethods rejects an + # empty method set. Guards direct construction. + return True + n_stable = sum(verdicts.values()) + if self.classification.decision == "consensus": + return n_stable == len(verdicts) + return n_stable * 2 > len(verdicts) + + def is_stable( + self, + change_series: RLEList[bool] | List[bool], + timestamps: List[float] | None = None, + ) -> bool: + """Combined stability verdict under the configured methods. + + ``timestamps`` are forwarded only to whichever enabled methods read + the time axis (``time``, ``slope_time`` -- see ``verdicts()``). With + only index-axis methods enabled, passing ``timestamps`` here has no + effect and the call classifies exactly as if they were omitted. + """ + return self.decide(self.verdicts(change_series, timestamps=timestamps)) + + def get_last_details(self) -> Dict[str, str]: + """One line per method from the last verdicts() call.""" + return self.last_details + + def slope( + self, + change_series: RLEList[bool] | List[bool], + timestamps: List[float] | None = None, + ) -> float: + """Change centroid: where in the series the changes sit, in [-0.5, +0.5]. + + The mean position of the changes, measured against the midpoint of the + range those changes could occupy and scaled by its half-span. -0.5 is + every change at the earliest countable position, 0 is uniform churn, + +0.5 is every change at the very end. + + With no usable timestamps the position is the index ``p``:: + + k = (p_bar - n/2) / (n - 2) + + With usable timestamps it is normalized time ``u``, measured against + the range still reachable once index 0 is excluded:: + + u = (t - t_first) / (t_last - t_first) + u_first = (t_1 - t_first) / (t_last - t_first) + k = (u_bar - (u_first + 1) / 2) / (1 - u_first) + + Substituting evenly spaced stamps collapses the second form into the + first, so the two axes agree exactly on uniformly stamped data. That + is what lets both slope methods share one threshold. + + Index 0 is excluded on both axes: the first value is always recorded as + a change, so counting it would drag every variable negative, a + perfectly static one included. On the time axis that exclusion is also + why the denominator is the achievable range and not the full span. + + The index form is the least-squares slope over the same series with its + data-free parts divided out -- for evenly spaced x the OLS denominator + is the constant n(n-1)(n-2)/12, and the numerator collapses to + m(p_bar - n/2) because only the change positions survive the binary y. + The two are related by ``k_OLS = k * 12m / n(n-1)``, a strictly + positive factor, so they never disagree on sign. Dropping the leading + ``m`` is the point: it is what makes k comparable between events + instead of scaling with how many changes happened to occur. + + Runs close in form, so an RLEList costs one pass over ``runs()`` with + no expansion. Returns 0.0 when the series is too short to have a span, + and -0.5 when nothing ever changed. + """ + return self._slope(change_series, timestamps)[0] + + def _slope( + self, + change_series: RLEList[bool] | List[bool], + timestamps: List[float] | None = None, + ) -> tuple[float, str]: + """``slope()`` plus the axis it actually used, for the reason string. + + The axis is not always the one configured: ``slope_time`` degrades to + the index axis on unusable timestamps, and a note that did not say so + would be misleading. + """ + n = len(change_series) + if n < 3: + return 0.0, "index" + # One narrowed local carries "use the time axis" and the stamps + # together, so the two can never drift apart. + stamps = timestamps + if stamps is not None and not _timestamps_usable(stamps, n): + stamps = None + if stamps is not None: + span = stamps[-1] - stamps[0] + u_first = (stamps[1] - stamps[0]) / span + # Every countable position shares one instant: no range to + # normalize against, so fall back rather than divide by zero. + if u_first >= 1.0: + stamps = None + runs = ( + change_series.runs() if isinstance(change_series, RLEList) + else ((value, 1) for value in change_series) + ) + position_sum, n_changes, position = 0.0, 0, 0 + for value, count in runs: + if value: + start = max(position, 1) # index 0 is excluded + length = position + count - start + if length > 0: + if stamps is not None: + # Per-element accumulation required: summing a multi-element + # run with np.sum() changes order of operations vs. processing + # individual elements, breaking bit-identical agreement between + # RLEList and plain-list code paths. + for i in range(start, start + length): + position_sum += float(stamps[i]) + else: + # sum of start .. start+length-1 + position_sum += length * start + length * (length - 1) // 2 + n_changes += length + position += count + if n_changes == 0: + return -0.5, "time" if stamps is not None else "index" + mean_position = position_sum / n_changes + if stamps is not None: + span = stamps[-1] - stamps[0] + u_bar = (mean_position - stamps[0]) / span + u_first = (stamps[1] - stamps[0]) / span + return (u_bar - (u_first + 1) / 2) / (1 - u_first), "time" + return (mean_position - n / 2) / (n - 2), "index" + def get_last_segment_means(self) -> List[float]: return self.segment_means @@ -137,5 +351,6 @@ def __call__( def __repr__(self) -> str: return ( f"StabilityClassifier(segment_threshs={self.segment_threshs}, " + f"classification={self.classification}, " f"segment_means={self.segment_means})" ) diff --git a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py index a66ea4b3..b6fd5d89 100644 --- a/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py +++ b/src/detectmatelibrary/utils/persistency/event_data_structures/trackers/stability/stability_tracker.py @@ -7,6 +7,7 @@ from detectmatelibrary.utils.persistency.rle_list import RLEList from ..base import SingleTracker, MultiTracker, EventTracker, Classification from .stability_classifier import StabilityClassifier +from .classification_methods import ClassificationMethods if TYPE_CHECKING: from detectmatelibrary.common.detector import CoreDetectorConfig @@ -40,22 +41,64 @@ def _strip_persist(detector_config: Any, method_id: str) -> Any: } +def _as_methods( + classification: "ClassificationMethods | Dict[str, Any] | None", +) -> ClassificationMethods: + """Accept the block as a model, a plain dict, or nothing. + + to_state() writes a dict and the config layer forwards a dict, so the + tracker has to take both without either caller converting first. + + Returns a copy when given a model instance: an ``EventStabilityTracker`` + shares one passed-in ``ClassificationMethods`` across every per-variable + tracker it creates, so storing the caller's instance as-is would let a + later in-place mutation of it silently change every variable already + built from it. + """ + if classification is None: + return ClassificationMethods() + if isinstance(classification, ClassificationMethods): + return classification.model_copy() + return ClassificationMethods(**classification) + + +def _classification_from_state(state: Dict[str, Any]) -> ClassificationMethods: + """The classification block for a state dict, old or new. + + Legacy snapshots predate the four-method split. Their `segmentation` + enum maps onto the two segment-threshold methods and `require_declining` + onto `slope_index`; their semantics were always AND, so they decide by + consensus. This is the only place the old names survive -- the config + layer rejects them outright. + """ + if "classification" in state: + return ClassificationMethods(**state["classification"]) + segmentation = state.get("segmentation", "count") + return ClassificationMethods( + index=segmentation in ("count", "both"), + time=segmentation in ("time", "both"), + slope_index=bool(state.get("require_declining", False)), + slope_threshold=state.get("incline_threshold", -0.05), + decision="consensus", + ) + + class SingleStabilityTracker(SingleTracker): """Tracks stability of a single feature.""" def __init__( self, min_samples: int = 3, - segmentation: Literal["count", "time", "both"] = "count", + classification: "ClassificationMethods | Dict[str, Any] | None" = None, add_value_fn: str = "default", detector_config: "CoreDetectorConfig | None" = None, ) -> None: self.min_samples = min_samples - self.segmentation = segmentation self.change_series: RLEList[bool] = RLEList() self.unique_set: Set[Any] = set() self.stability_classifier: StabilityClassifier = StabilityClassifier( segment_thresholds=[1.1, 0.3, 0.1, 0.01], + classification=_as_methods(classification), ) # ponytail: O(N) timestamps; switch to fixed-width time buckets if # this ever runs unbounded/streaming. @@ -65,7 +108,7 @@ def __init__( self.extra_state: Dict[str, Any] = {} self.add_value_fn = add_value_fn self.detector_config = detector_config - # Transient: set by _is_stable() for classify()'s reason string. Not + # Transient: set by _is_stable() as classify()'s reason string. Not # persisted -- it is derived from change_series on every classify(). self._stability_note: str = "" self._value_fn: Callable[[Any], None] = self._default_add_value @@ -77,6 +120,25 @@ def __init__( detector = detector_cls() self._value_fn = partial(detector.add_value, self) + @property + def classification(self) -> ClassificationMethods: + """The classification methods in force, owned by the classifier. + + A property rather than a second attribute: the tracker reads it at + ingest time (to decide whether to collect timestamps) and the + classifier reads it at classify() time, and two copies would let + those two drift apart. Reassignable between classify() calls -- the + methods are read when classify() runs, never when values arrive, so + several verdicts can be taken from one ingest by swapping this. + """ + return self.stability_classifier.classification + + @classification.setter + def classification( + self, value: "ClassificationMethods | Dict[str, Any] | None" + ) -> None: + self.stability_classifier.classification = _as_methods(value) + def _default_add_value(self, value: Any) -> None: """Default value semantics: one set entry per whole value.""" before = len(self.unique_set) @@ -90,11 +152,16 @@ def add_value(self, value: Any, timestamp: float | None = None) -> None: detector's ``add_value``. Timestamp bookkeeping stays here so ``timestamps`` cannot drift from ``change_series``: a detector may record nothing for a value (ValueRangeDetector on non-numeric input), and a - length mismatch silently demotes the variable to count segmentation. + length mismatch silently leaves that value off the time axis, demoting + the variable to index-based classification. """ before = len(self.change_series) self._value_fn(value) - if self.segmentation != "count" and timestamp is not None and len(self.change_series) > before: + if ( + self.classification.needs_timestamps + and timestamp is not None + and len(self.change_series) > before + ): self.timestamps.append(float(timestamp)) def classify(self) -> Classification: @@ -114,62 +181,40 @@ def classify(self) -> Classification: type="RANDOM", reason=f"Unique set size equals number of samples ({len(self.change_series)})" ) - elif self._is_stable(): - return Classification( - type="STABLE", - reason=( - f"{self._stability_note} are below segment thresholds: " - f"{self.stability_classifier.get_segment_thresholds()}" - ) - ) - else: - return Classification( - type="UNSTABLE", - reason=( - f"{self._stability_note} exceed segment thresholds: " - f"{self.stability_classifier.get_segment_thresholds()}" - ) - ) + stable = self._is_stable() + return Classification( + type="STABLE" if stable else "UNSTABLE", + reason=self._stability_note, + ) def _is_stable(self) -> bool: - """Stability verdict under the configured segmentation. + """Stability verdict under the configured classification methods. - Sets ``_stability_note`` for ``classify()``'s reason string. - - ``both`` runs the count pass and the time pass over the same - change series and requires both. Neither segmentation subsumes - the other -- a variable that churns in a burst and then settles is - count-UNSTABLE but time-STABLE, and one whose late churn is buried - under a dense settled tail is the reverse -- so the conjunction is - strictly stricter than either input. - - Deliberately not short-circuited: both passes always run so the - note carries both mean vectors, which is what anyone debugging a - ``both`` verdict needs. Costs one extra O(runs x n_segments) scan. + Builds ``_stability_note``, which is ``classify()``'s whole reason + string. The note names every enabled method, what it found, and how + the decision rule resolved -- with four selectable methods and two + decision rules, naming only the verdict would leave a reader unable + to tell which method drove it. """ - clf, ts = self.stability_classifier, self._aligned_timestamps() - if self.segmentation != "both": - stable = clf.is_stable(self.change_series, timestamps=ts) - self._stability_note = f"Segment means of change series {clf.get_last_segment_means()}" - return stable - count_stable = clf.is_stable(self.change_series) - # Snapshot now, not after the time pass: is_stable() rebinds - # clf.segment_means to a fresh list on every call, so calling - # get_last_segment_means() after the time pass below would return - # the time means for both halves of the note instead of the count - # means it is meant to capture here. - count_means = clf.get_last_segment_means() - time_stable = clf.is_stable(self.change_series, timestamps=ts) - self._stability_note = ( - f"Segment means of change series: count {count_means}, " - f"time {clf.get_last_segment_means()}" + clf = self.stability_classifier + verdicts = clf.verdicts(self.change_series, timestamps=self._aligned_timestamps()) + n_stable, n_total = sum(verdicts.values()), len(verdicts) + verdict = clf.decide(verdicts) + details = clf.get_last_details() + self._stability_note = "; ".join( + [details[name] for name in verdicts] + + [f"decision={clf.classification.decision} ({n_stable}/{n_total}) -> " + f"{'STABLE' if verdict else 'UNSTABLE'}"] ) - return count_stable and time_stable + return verdict def _aligned_timestamps(self) -> List[float] | None: - """Timestamps to classify with, or None to fall back to count - segments.""" - if self.segmentation != "count" and len(self.timestamps) == len(self.change_series): + """Timestamps to classify with, or None to fall back to the index + axis.""" + if ( + self.classification.needs_timestamps + and len(self.timestamps) == len(self.change_series) + ): return self.timestamps return None @@ -180,7 +225,7 @@ def to_state(self) -> Dict[str, Any]: "type": self.__class__.__name__, "module": self.__class__.__module__, "min_samples": self.min_samples, - "segmentation": self.segmentation, + "classification": self.classification.model_dump(), "timestamps": self.timestamps, "add_value_fn": self.add_value_fn, "detector_config": self.detector_config, @@ -192,13 +237,20 @@ def to_state(self) -> Dict[str, Any]: @classmethod def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": - """Restore tracker from a state dict produced by to_state().""" - # Every optional key is read with .get(): a snapshot old enough to still - # carry the removed `expand_value` predates `add_value_fn` too, so - # indexing here would KeyError on exactly the states this tolerance is for. + """Restore tracker from a state dict produced by to_state(). + + Every optional key is read with .get(): a snapshot old enough to still + carry the removed `expand_value` predates `add_value_fn` too, so + indexing here would KeyError on exactly the states this tolerance is + for. The same applies to the classification block -- snapshots written + before the four-method split carry `segmentation` / `require_declining` + / `incline_threshold` instead, and _classification_from_state + translates them. + """ + classification = _classification_from_state(state) tracker = cls( min_samples=state["min_samples"], - segmentation=state.get("segmentation", "count"), + classification=classification, add_value_fn=state.get("add_value_fn", "default"), detector_config=state.get("detector_config"), ) @@ -208,8 +260,12 @@ def from_state(cls, state: Dict[str, Any]) -> "SingleStabilityTracker": tracker.unique_set = { tuple(v) if isinstance(v, list) else v for v in state["unique_set"] } + # Rebuilding the classifier drops the one __init__ made, so the block + # is passed from the local -- reading tracker.classification here would + # read through the very object being replaced. tracker.stability_classifier = StabilityClassifier( - segment_thresholds=state["segment_thresholds"] + segment_thresholds=state["segment_thresholds"], + classification=classification, ) tracker.timestamps = [float(t) for t in state.get("timestamps", [])] tracker.extra_state = state.get("extra_state", {}) @@ -221,7 +277,7 @@ def __repr__(self) -> str: unique_set_str = "{" + ", ".join(map(str, list_preview_str(self.unique_set))) + "}" RLE_str = list_preview_str(self.change_series.runs()) return ( - f"{self.__class__.__name__}(classification={self.classify()}, change_series={series_str}, " + f"{self.__class__.__name__}(verdict={self.classify()}, change_series={series_str}, " f"unique_set={unique_set_str}, RLE={RLE_str})" ) @@ -251,16 +307,15 @@ class EventStabilityTracker(EventTracker): def __init__( self, converter_function: Callable[[Any], Any] = lambda x: x, - segmentation: Literal["count", "time", "both"] = "count", + classification: "ClassificationMethods | Dict[str, Any] | None" = None, add_value_fn: str = "default", - detector_config: "CoreDetectorConfig | None" = None - + detector_config: "CoreDetectorConfig | None" = None, ) -> None: self.multi_tracker: MultiStabilityTracker # for type hinting def make_tracker() -> SingleStabilityTracker: return SingleStabilityTracker( - segmentation=segmentation, + classification=classification, add_value_fn=add_value_fn, detector_config=detector_config, ) diff --git a/src/detectmatelibrary/utils/sequence_encoding.py b/src/detectmatelibrary/utils/sequence_encoding.py new file mode 100644 index 00000000..9a468456 --- /dev/null +++ b/src/detectmatelibrary/utils/sequence_encoding.py @@ -0,0 +1,76 @@ +"""Encode event sequences and count vectors as EventPersistency keys. + +`EventPersistency` keys events by ID, so detectors whose model is a set of +sequences store each sequence as a string key in `events_seen` and get save, +load and auto-load for free. Shared here rather than in any one detector so +detectors never have to import from each other. +""" + +from typing import List, Sequence + +from detectmatelibrary import schemas +from detectmatelibrary.tools.logging import logger +from detectmatelibrary.utils.persistency import EventPersistency + +_SEQUENCE_SEPARATOR = "\x1f" + + +def encode_sequence(sequence: Sequence[int]) -> str: + """Encode a sequence of integers as a persistency key.""" + return _SEQUENCE_SEPARATOR.join(str(event_id) for event_id in sequence) + + +def decode_sequence(encoded: str) -> tuple[int, ...]: + """Inverse of `encode_sequence`.""" + return tuple(int(event_id) for event_id in encoded.split(_SEQUENCE_SEPARATOR)) + + +def build_count_vec(input_: List[schemas.ParserSchema]) -> tuple[int, ...]: + """Count how often each EventID occurs in a window, indexed by EventID.""" + sequence, n = [0], 0 + for in_ in input_: + event = in_["EventID"] + if n < event: + for _ in range(n, event): + sequence.append(0) + n = event + sequence[event] += 1 + + return tuple(sequence) + + +def encode_count_vec(window_size: int, count_vec: tuple[int, ...]) -> str: + """Encode a count vector as a persistency key. + + The window size leads the key so restored state can be compared against the + configured window: a count vector's length is max(EventID) + 1, which says + nothing about the window it was counted over. + """ + return encode_sequence((window_size, *count_vec)) + + +def decode_count_vec(encoded: str) -> tuple[int, tuple[int, ...]]: + """Inverse of `encode_count_vec`, as (window size, count vector).""" + window_size, *count_vec = decode_sequence(encoded) + return window_size, tuple(count_vec) + + +def warn_on_window_size_mismatch( + name: str, event_persistency: EventPersistency, window_size: int +) -> None: + """Warn when restored count vectors were trained at another window size. + + Count vectors are only comparable within the window they were + counted over, so every restored vector would miss and detection + would degrade into a stream of false positives. + """ + restored = event_persistency.get_events_seen() + if not restored: + return + trained, _ = decode_count_vec(str(next(iter(restored)))) + if trained != window_size: + logger.warning( + f"[{name}] restored state was trained with window_size {trained}, but " + f"window_size is {window_size}. Count vectors from different windows " + "are not comparable — expect false positives until retrained." + ) diff --git a/tests/test_common/test_auto_config_params.py b/tests/test_common/test_auto_config_params.py new file mode 100644 index 00000000..e02a0d6e --- /dev/null +++ b/tests/test_common/test_auto_config_params.py @@ -0,0 +1,101 @@ +"""The auto_config_params block: parsing, round-trip, and strictness.""" + +import warnings + +import pytest +from pydantic import ValidationError + +from detectmatelibrary.common._config import AutoConfigParams, BasicConfig +from detectmatelibrary.common._config._compile import MissingParamsWarning +from detectmatelibrary.common.alert_aggregator import CoreAlertAggregatorConfig +from detectmatelibrary.common.detector import CoreDetectorConfig +from detectmatelibrary.common.parser import CoreParserConfig + +CONFIG_CLASSES = (CoreParserConfig, CoreDetectorConfig, CoreAlertAggregatorConfig) + + +class _Params(AutoConfigParams): + knob: int = 1 + + +class _Config(CoreDetectorConfig): + method_type: str = "test_detector" + auto_config_params: _Params = _Params() + + +def _wrap(entry: dict) -> dict: + return {"detectors": {"TestDetector": entry}} + + +def _from_dict(**entry: object) -> _Config: + return _Config.from_dict( + _wrap({"method_type": "test_detector", "auto_config": True, **entry}), + "TestDetector", + ) + + +def test_block_round_trips(): + """YAML -> pydantic -> YAML, staying out of the operational params + block.""" + cfg = _from_dict(auto_config_params={"knob": 7}) + assert cfg.auto_config_params.knob == 7 + + dumped = cfg.to_dict(method_id="TestDetector")["detectors"]["TestDetector"] + assert dumped["auto_config_params"] == {"knob": 7} + assert "knob" not in dumped.get("params", {}) + + +def test_default_block_is_not_emitted(): + """A config that never touches auto-config serializes exactly as before. + + Also covers the component types that inherit the block empty: adding it + to the shared base must not add a key to anyone's YAML. + """ + dumped = _Config().to_dict(method_id="TestDetector")["detectors"]["TestDetector"] + assert "auto_config_params" not in dumped + + for config_cls in CONFIG_CLASSES: + config = config_cls() + dumped = config.to_dict(method_id="M")[config.component_type]["M"] + assert "auto_config_params" not in dumped + + +def test_unknown_key_in_block_is_rejected(): + """Extra='forbid' reaches every component type, including those that + declare no fields in the block.""" + with pytest.raises(ValidationError): + _from_dict(auto_config_params={"nope": 1}) + + for config_cls in CONFIG_CLASSES: + with pytest.raises(ValidationError): + config_cls(auto_config_params={"nope": 1}) + + +def test_auto_param_under_params_is_rejected(): + """The clean break: the old flat spelling is an error, not a silent no- + op.""" + with pytest.raises(ValidationError): + _from_dict(params={"knob": 7}) + + +def test_block_alone_counts_as_data(): + """auto_config_params is real configuration and must not trip + MissingParamsWarning.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingParamsWarning) + _Config.from_dict( + _wrap({ + "method_type": "test_detector", + "auto_config": False, + "auto_config_params": {"knob": 7}, + }), + "TestDetector", + ) + + +def test_block_is_declared_on_the_shared_base(): + """Not detector-only: the block sits beside `auto_config` on BasicConfig, + since `auto_config` and `Component.configure()` are both declared there.""" + assert "auto_config_params" in BasicConfig.model_fields + for config_cls in CONFIG_CLASSES: + assert isinstance(config_cls().auto_config_params, AutoConfigParams) diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index d83c5960..45ee65ab 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -4,13 +4,13 @@ MissingParamsWarning, TypeNotFoundError, MethodTypeNotMatch, - AutoConfigWarning, ) from detectmatelibrary.common._config._formats import EventsConfig, _EventConfig from detectmatelibrary.common._config import BasicConfig from pydantic import ValidationError from tests.test_data import TEST_CONFIG import pytest +import warnings import yaml @@ -81,11 +81,16 @@ def test_process_auto_config_false(self): config_test, method_id="detector_wrong", component_type="detectors" )) - def test_process_auto_config_warning(self): - with pytest.warns(AutoConfigWarning): - ConfigMethods.process(ConfigMethods.get_method( + def test_process_keeps_params_under_auto_config(self): + """params are operational and survive the configure phase, so + auto_config: True alongside params is no longer suspicious.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + config = ConfigMethods.process(ConfigMethods.get_method( config_test, method_id="detector_weird", component_type="detectors" )) + assert config["auto_config"] is True + assert config["hello"] == "a" class TestParamsFormat: diff --git a/tests/test_common/test_core.py b/tests/test_common/test_core.py index e5822c34..2ee3c0a2 100644 --- a/tests/test_common/test_core.py +++ b/tests/test_common/test_core.py @@ -28,6 +28,7 @@ class MockConfigWithTraining(CoreConfig): "method_type": "default_method_type", "component_type": "default_type", "auto_config": False, + "auto_config_params": {}, "start_id": 10, "data_use_training": None, "data_use_configure": None, diff --git a/tests/test_common/test_core_federation.py b/tests/test_common/test_core_federation.py new file mode 100644 index 00000000..826fe406 --- /dev/null +++ b/tests/test_common/test_core_federation.py @@ -0,0 +1,156 @@ +from detectmatelibrary.common._core_op._fed_component import IncompatibleFed +from detectmatelibrary.common.core import CoreComponent + +import struct + +import pytest + + +class DummyComponent(CoreComponent): + pass + + +class DummyComponent2(CoreComponent): + pass + + +class TestJoinOp: + def test_add(self) -> None: + component1 = CoreComponent(name="comp_1") + component2 = CoreComponent(name="comp_2") + component3 = CoreComponent(name="comp_3") + + component2 + component3 + assert len(component2._components) == 2 + assert component3._components == component2._components + + component1 = component1 + component3 + assert len(component1._components) == 3 + assert component1._components == component2._components + assert component1._components == component3._components + + def test_sub(self) -> None: + component1 = CoreComponent(name="comp_1") + component2 = CoreComponent(name="comp_2") + component3 = CoreComponent(name="comp_3") + + (component1 + component2 + component3) - component3 + assert len(component2._components) == 2 + assert component1._components == component2._components + assert component3._components == {component3} + + def test_incompatible(self) -> None: + component1 = DummyComponent2(name="comp_1") + component2 = DummyComponent(name="comp_2") + + with pytest.raises(IncompatibleFed): + component1 + component2 + with pytest.raises(IncompatibleFed): + component1 - component2 + + def test_stack(self) -> None: + component1 = CoreComponent(name="comp_1") + component2 = CoreComponent(name="comp_2") + component3 = CoreComponent(name="comp_3") + component4 = CoreComponent(name="comp_4") + + component1.stack(component2) + assert len(component1._components) == 2 + assert component1._components != component2._components + + component1.stack([component3, component4]) + assert len(component1._components) == 4 + + +class DummyAppendList(CoreComponent): + def __init__( + self, elems: list[str], name: str = "test", *args, **kwargs + ) -> None: + super().__init__(name, *args, **kwargs) + self.elems = elems + + def aggregate_strategy(self, components): + final_list = [] + for component in components: + final_list.extend(component.elems) + + final_list = list(set(final_list)) + for component in components: + component.elems = final_list + + def to_binary(self): + return struct.pack(f">{len(self.elems)}h", *self.elems) + + def from_binary(self, binary): + num_ints = len(binary) // 2 + elems = list(struct.unpack(f">{num_ints}h", binary)) + return DummyAppendList(elems=elems) + + +class DummyAppendListEmpty(CoreComponent): + def __init__( + self, elems: list[str], name: str = "test", *args, **kwargs + ) -> None: + super().__init__(name, *args, **kwargs) + self.elems = elems + + +class TestFedComponent: + def test_basic_aggregation(self) -> None: + comp1 = DummyAppendList(elems=[1, 2]) + comp2 = DummyAppendList(elems=[3, 4]) + + comp1.aggregate() + assert set(comp1.elems) == {1, 2} + + (comp1 + comp2).aggregate() + assert set(comp1.elems) == {1, 2, 3, 4} + + def test_stack_basic_aggregation(self) -> None: + comp1 = DummyAppendList(elems=[1, 2]) + comp2 = DummyAppendList(elems=[3, 4]) + comp3 = DummyAppendList(elems=[5]) + + comp1.stack([comp2, comp3]) + comp1.aggregate(unstack=True) + assert set(comp1.elems) == {1, 2, 3, 4, 5} + assert len(comp1._components) == 1 + + comp1.stack([comp2, comp3]) + comp1.aggregate(unstack=False) + assert set(comp1.elems) == {1, 2, 3, 4, 5} + assert len(comp1._components) == 3 + + def test_sanity_check(self) -> None: + comp1 = DummyAppendList(elems=[1, 2]) + comp2 = comp1.from_binary(comp1.to_binary()) + + assert comp2.elems == [1, 2] + + def test_stack_binary_aggregation(self) -> None: + comp1 = DummyAppendList(elems=[1, 2]) + comp2 = DummyAppendList(elems=[3, 4]) + comp3 = DummyAppendList(elems=[5]) + + comp1.stack([comp2.to_binary(), comp3.to_binary()]) + comp1.aggregate(unstack=True) + assert set(comp1.elems) == {1, 2, 3, 4, 5} + + comp1.stack([comp2.to_binary(), comp3.to_binary()]) + output = comp1.aggregate(unstack=True) + assert set(comp1.elems) == {1, 2, 3, 4, 5} + assert isinstance(output, bytes) + + def test_empty_feed_fields(self) -> None: + comp1 = DummyAppendListEmpty(elems=[1, 2]) + comp2 = DummyAppendListEmpty(elems=[3, 4]) + comp3 = DummyAppendListEmpty(elems=[5]) + + with pytest.warns(UserWarning): + comp1.to_binary() + + with pytest.warns(UserWarning): + comp1.from_binary(b"") + + with pytest.warns(UserWarning): + comp1.aggregate_strategy({comp1, comp2, comp3}) diff --git a/tests/test_detectors/test_auto_config_params_survive.py b/tests/test_detectors/test_auto_config_params_survive.py new file mode 100644 index 00000000..ca51152f --- /dev/null +++ b/tests/test_detectors/test_auto_config_params_survive.py @@ -0,0 +1,175 @@ +"""set_configuration writes only its outputs. + +Every field on a detector config other than `events` and `auto_config` is +operator input and must read back unchanged after the configure phase. This +is the regression test for auto-config inputs being silently reset. +""" + +from detectmatelibrary.common._config._formats import EventsConfig +from detectmatelibrary.detectors.event_sequence_detector import ( + EventSequenceDetector, + EventSequenceDetectorConfig, + SequenceAutoConfigParams, +) +from detectmatelibrary.detectors.new_event_detector import ( + NewEventDetector, + NewEventDetectorConfig, +) +from detectmatelibrary.detectors.new_value_combo_detector import ( + ComboAutoConfigParams, + NewValueComboDetector, + NewValueComboDetectorConfig, +) +from detectmatelibrary.detectors.new_value_detector import ( + NewValueDetector, + NewValueDetectorConfig, +) +from detectmatelibrary.common.variable_detector import VariableAutoConfigParams + + +def _schema(event_id: int, level: str, log_id: str): + return { + "logID": log_id, + "EventID": event_id, + "template": "user <*> from <*>", + "variables": ["alice", "10.0.0.1"], + "logFormatVariables": {"user": "alice", "src": "10.0.0.1", "level": level}, + } + + +_AUTO = dict( + use_stable_vars=True, + use_static_vars=True, + classification=dict(index=False, time=True, slope_index=True), + timestamp_variable="level", + timestamp_format="%y%m%d %H%M%S", +) + +_STREAM = [_schema(1, f"081109 2036{i:02d}", str(i)) for i in range(20)] + + +def _assert_auto_params_intact(config): + auto = config.auto_config_params + assert auto.use_stable_vars is True + assert auto.use_static_vars is True + assert auto.classification.enabled == ("time", "slope_index") + assert auto.timestamp_variable == "level" + assert auto.timestamp_format == "%y%m%d %H%M%S" + + +def test_new_value_detector_keeps_auto_config_params(): + detector = NewValueDetector( + name="NewValueDetector", + config=NewValueDetectorConfig( + parser="MyParser", + auto_config_params=VariableAutoConfigParams(**_AUTO), + ), + ) + for record in _STREAM: + detector.configure(record) + detector.set_configuration() + + _assert_auto_params_intact(detector.config) + assert detector.config.parser == "MyParser" + assert detector.config.auto_config is False + assert isinstance(detector.config.events, EventsConfig) + + +def test_combo_detector_keeps_auto_config_params(): + detector = NewValueComboDetector( + name="NewValueComboDetector", + config=NewValueComboDetectorConfig( + parser="MyParser", + auto_config_params=ComboAutoConfigParams(**_AUTO), + ), + ) + for record in _STREAM: + detector.configure(record) + detector.set_configuration() + + _assert_auto_params_intact(detector.config) + assert detector.config.parser == "MyParser" + assert detector.config.auto_config is False + assert isinstance(detector.config.events, EventsConfig) + + +def test_new_event_detector_keeps_operator_settings(): + detector = NewEventDetector( + name="NewEventDetector", + config=NewEventDetectorConfig(parser="MyParser", data_use_training=17), + ) + for record in _STREAM: + detector.configure(record) + detector.set_configuration() + + assert detector.config.parser == "MyParser" + assert detector.config.data_use_training == 17 + assert detector.config.auto_config is False + assert isinstance(detector.config.events, EventsConfig) + + +def test_event_sequence_detector_keeps_operator_settings(): + # Same shape as test_unstable_stream_generates_no_instance in + # test_event_sequence_detector.py: every EventID is unique, so every + # candidate window fills and clears min_samples but its sequences never + # repeat -- no candidate is ever classified STABLE/STATIC. That drives + # the no-stable-candidate early-return branch this task rewrote, which + # is the widest-blast-radius site: it is not a VariableDetector, so + # before this task it restored only `persist` by hand and silently + # reset every other operator field. + detector = EventSequenceDetector( + name="EventSequenceDetector", + config=EventSequenceDetectorConfig( + parser="MyParser", + data_use_configure=5, + data_use_training=1, + use_config_data_as_training=False, + auto_config_params=SequenceAutoConfigParams( + min_window_size=3, + max_window_size=6, + ), + ), + ) + for i, event_id in enumerate(range(40)): + detector.configure(_schema(event_id, "081109 203600", str(i))) + detector.set_configuration() + + assert detector.config.fixed_window_size is None + assert detector.config.parser == "MyParser" + assert detector.config.data_use_configure == 5 + assert detector.config.data_use_training == 1 + assert detector.config.use_config_data_as_training is False + assert detector.config.auto_config_params.min_window_size == 3 + assert detector.config.auto_config_params.max_window_size == 6 + assert detector.config.auto_config is False + assert isinstance(detector.config.events, EventsConfig) + + +def test_event_sequence_detector_keeps_auto_config_params(): + """The sequence detector writes fixed_window_size, not a fresh config.""" + from detectmatelibrary.detectors.event_sequence_detector import ( + EventSequenceDetector, + EventSequenceDetectorConfig, + SequenceAutoConfigParams, + ) + + detector = EventSequenceDetector( + name="EventSequenceDetector", + config=EventSequenceDetectorConfig( + parser="MyParser", + auto_config_params=SequenceAutoConfigParams( + min_window_size=2, max_window_size=4 + ), + ), + ) + for record in _STREAM: + detector.configure(record) + detector.set_configuration() + + auto = detector.config.auto_config_params + assert auto.min_window_size == 2 + assert auto.max_window_size == 4 + assert detector.config.parser == "MyParser" + assert detector.config.fixed_window_size == 4 + assert detector.config.auto_config is False + assert isinstance(detector.config.events, EventsConfig) diff --git a/tests/test_detectors/test_event_sequence_detector.py b/tests/test_detectors/test_event_sequence_detector.py index 461069ec..d4b872cf 100644 --- a/tests/test_detectors/test_event_sequence_detector.py +++ b/tests/test_detectors/test_event_sequence_detector.py @@ -11,7 +11,7 @@ from pydantic import ValidationError from detectmatelibrary.detectors.event_sequence_detector import EventSequenceDetector, \ - EventSequenceDetectorConfig, BufferMode + EventSequenceDetectorConfig, SequenceAutoConfigParams, BufferMode from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From import detectmatelibrary.schemas as schemas @@ -72,7 +72,10 @@ def test_default_initialization(self): assert hasattr(detector, "persistency") # unconfigured until auto-config picks a length assert detector.config.fixed_window_size is None - assert (detector.config.min_window_size, detector.config.max_window_size) == (2, 10) + assert ( + detector.config.auto_config_params.min_window_size, + detector.config.auto_config_params.max_window_size, + ) == (2, 10) def test_custom_config_initialization(self): detector = EventSequenceDetector(name="CustomInit", config=config) @@ -277,7 +280,9 @@ def test_short_configure_phase_skips_unfilled_candidates(self): name="ShortConfig", config=EventSequenceDetectorConfig( data_use_configure=6, data_use_training=1, - min_window_size=2, max_window_size=8, + auto_config_params=SequenceAutoConfigParams( + min_window_size=2, max_window_size=8, + ), ), ) @@ -295,8 +300,10 @@ def test_set_configuration_preserves_user_config(self): data_use_configure=6, data_use_training=10, use_config_data_as_training=False, - min_window_size=2, - max_window_size=8, + auto_config_params=SequenceAutoConfigParams( + min_window_size=2, + max_window_size=8, + ), ), ) @@ -307,17 +314,22 @@ def test_set_configuration_preserves_user_config(self): assert detector.config.parser == "MySequenceParser" assert detector.config.data_use_training == 10 assert detector.config.use_config_data_as_training is False - assert (detector.config.min_window_size, detector.config.max_window_size) == (2, 8) + assert ( + detector.config.auto_config_params.min_window_size, + detector.config.auto_config_params.max_window_size, + ) == (2, 8) def test_configure_windows_follow_config_changes(self): """_configure_windows is built lazily, so changing the range after construction must not raise.""" detector = EventSequenceDetector( name="LateCandidates", - config=EventSequenceDetectorConfig(min_window_size=2, max_window_size=3), + config=EventSequenceDetectorConfig( + auto_config_params=SequenceAutoConfigParams(min_window_size=2, max_window_size=3), + ), ) - detector.config.min_window_size = 4 - detector.config.max_window_size = 5 + detector.config.auto_config_params.min_window_size = 4 + detector.config.auto_config_params.max_window_size = 5 detector.configure(_make_schema(1)) @@ -329,7 +341,8 @@ def test_fixed_window_size_skips_auto_config(self): name="FixedWins", config=EventSequenceDetectorConfig( data_use_configure=5, data_use_training=1, - min_window_size=4, max_window_size=6, fixed_window_size=2, + auto_config_params=SequenceAutoConfigParams(min_window_size=4, max_window_size=6), + fixed_window_size=2, ), ) @@ -347,7 +360,7 @@ def test_no_stable_window_size_generates_no_instance(self): config=EventSequenceDetectorConfig( data_use_configure=5, data_use_training=1, # no window of 8+ can fill within a 5-event configure phase - min_window_size=8, max_window_size=10, + auto_config_params=SequenceAutoConfigParams(min_window_size=8, max_window_size=10), ), ) @@ -367,7 +380,9 @@ def test_unstable_stream_generates_no_instance(self): clears min_samples, but the sequences never settle.""" detector = EventSequenceDetector( name="Unstable", - config=EventSequenceDetectorConfig(min_window_size=2, max_window_size=4), + config=EventSequenceDetectorConfig( + auto_config_params=SequenceAutoConfigParams(min_window_size=2, max_window_size=4), + ), ) for i, event_id in enumerate(range(30)): # every event ID unique @@ -513,7 +528,8 @@ def test_restored_state_disables_auto_config(self): config=EventSequenceDetectorConfig( data_use_configure=5, data_use_training=1, - min_window_size=4, max_window_size=5, # 3 deliberately excluded + # 3 deliberately excluded + auto_config_params=SequenceAutoConfigParams(min_window_size=4, max_window_size=5), persist=PersistConfig(path=base_path, auto_load=True), ), ) @@ -536,8 +552,7 @@ def test_persist_survives_empty_configuration(self): data_use_configure=5, data_use_training=1, # no window of 8+ can fill within a 5-event configure phase - min_window_size=8, - max_window_size=10, + auto_config_params=SequenceAutoConfigParams(min_window_size=8, max_window_size=10), persist=PersistConfig(path=base_path), ), ) @@ -582,14 +597,39 @@ def test_zero_fixed_window_size_rejected(self): def test_zero_min_window_size_rejected(self): with pytest.raises(ValidationError): - EventSequenceDetectorConfig(min_window_size=0) + SequenceAutoConfigParams(min_window_size=0) def test_inverted_window_range_rejected(self): with pytest.raises(ValidationError): - EventSequenceDetectorConfig(min_window_size=5, max_window_size=4) + SequenceAutoConfigParams(min_window_size=5, max_window_size=4) def test_removed_fields_rejected(self): """Extra='forbid': configs written for the old field names must fail loudly rather than silently run with defaults.""" with pytest.raises(ValidationError): EventSequenceDetectorConfig(max_sequence_length=3) + + def test_auto_config_params_round_trip(self): + block = {"min_window_size": 3, "max_window_size": 7} + source = { + "detectors": { + "EventSequenceDetector": { + "method_type": "event_sequence_detector", + "auto_config": True, + "auto_config_params": block, + # Explicit (rather than omitted) so the first from_dict + # already coerces `events` to EventsConfig -- otherwise it + # stays the bare-dict field default and the round-trip + # equality below fails on that unrelated field, not on + # auto_config_params (pydantic does not validate/coerce + # field defaults, only explicit constructor input). + "events": {}, + } + } + } + config = EventSequenceDetectorConfig.from_dict(source, "EventSequenceDetector") + dumped = config.to_dict(method_id="EventSequenceDetector") + entry = dumped["detectors"]["EventSequenceDetector"] + assert block.items() <= entry["auto_config_params"].items() + assert not set(block) & set(entry.get("params", {})) + assert EventSequenceDetectorConfig.from_dict(dumped, "EventSequenceDetector") == config diff --git a/tests/test_detectors/test_new_value_combo_detector.py b/tests/test_detectors/test_new_value_combo_detector.py index cc1f319e..cd4d5c01 100644 --- a/tests/test_detectors/test_new_value_combo_detector.py +++ b/tests/test_detectors/test_new_value_combo_detector.py @@ -2,6 +2,10 @@ from detectmatelibrary.detectors.new_value_combo_detector import ( NewValueComboDetector, NewValueComboDetectorConfig, + ComboAutoConfigParams, +) +from detectmatelibrary.utils.persistency.event_data_structures.trackers import ( + ClassificationMethods, ) from detectmatelibrary.utils.data_buffer import BufferMode from detectmatelibrary.common._config import generate_detector_config @@ -20,7 +24,7 @@ "CustomInit": { "method_type": "new_value_combo_detector", "auto_config": False, - "params": { + "auto_config_params": { "max_combo_size": 4 }, "events": { @@ -37,7 +41,7 @@ "MultipleDetector": { "method_type": "new_value_combo_detector", "auto_config": False, - "params": { + "auto_config_params": { "max_combo_size": 2 }, "events": { @@ -72,7 +76,7 @@ def test_custom_config_initialization(self): detector = NewValueComboDetector(name="CustomInit", config=config) assert detector.name == "CustomInit" - assert detector.config.max_combo_size == 4 + assert detector.config.auto_config_params.max_combo_size == 4 class TestNewValueComboDetectorTraining: @@ -291,7 +295,7 @@ def test_set_configuration_updates_config(self): # Verify config was updated assert detector.config.events is not None - assert detector.config.max_combo_size == 2 + assert detector.config.auto_config_params.max_combo_size == 2 def test_configuration_workflow(self): """Test complete configuration workflow like in notebook.""" @@ -363,7 +367,7 @@ def test_set_configuration_with_combo_size(self): detector.set_configuration(max_combo_size=4) # Verify max_combo_size was updated - assert detector.config.max_combo_size == 4 + assert detector.config.auto_config_params.max_combo_size == 4 def test_configuration_with_no_stable_variables(self): """Test configuration when no stable variables are found.""" @@ -582,20 +586,20 @@ def test_audit_log_anomalies(self): assert detected_ids == {"1859", "1862", "1865", "1866"} -class TestNewValueComboDetectorSegmentationConfigPreservation: - """set_configuration() reassigns self.config twice (pass 1: combo - candidates, pass 2: final selection), each from a freshly generated config - dict whose params only ever carry max_combo_size. +class TestNewValueComboDetectorClassificationConfigPreservation: + """auto_config_params survive set_configuration untouched. - stability_segmentation, timestamp_variable and timestamp_format must - survive both reassignments, the same way persist already does. + The configure phase writes only `events` and `auto_config`; every other + field on the config is operator input. """ - def test_segmentation_fields_survive_set_configuration(self): + def test_classification_fields_survive_set_configuration(self): cfg = NewValueComboDetectorConfig( - stability_segmentation="time", - timestamp_variable="level", - timestamp_format="%y%m%d %H%M%S", + auto_config_params=ComboAutoConfigParams( + classification=ClassificationMethods(index=False, time=True), + timestamp_variable="level", + timestamp_format="%y%m%d %H%M%S", + ), ) detector = NewValueComboDetector(config=cfg, name="NewValueComboDetector") assert detector.config.auto_config is True # the default (set_configuration-first) path @@ -616,29 +620,34 @@ def test_segmentation_fields_survive_set_configuration(self): detector.set_configuration(max_combo_size=2) - assert detector.config.stability_segmentation == "time" - assert detector.config.timestamp_variable == "level" - assert detector.config.timestamp_format == "%y%m%d %H%M%S" + assert detector.config.auto_config_params.classification.enabled == ("time",) + assert detector.config.auto_config_params.timestamp_variable == "level" + assert detector.config.auto_config_params.timestamp_format == "%y%m%d %H%M%S" -class TestNewValueComboDetectorSegmentationCombos: - """The combo-stability pass must honour stability_segmentation too. +class TestNewValueComboDetectorClassificationCombos: + """The combo-stability pass must honour the classification block too. auto_conf_persistency_combos is built directly in __init__ rather than from the _event_data_kwargs hook, and its re-ingest loop in set_configuration - calls ingest_event itself -- so both halves of the flag (the tracker kwarg - and the per-record timestamp) have to be wired up explicitly. A flag that - reaches the first-pass trackers but not the combo trackers is worse than no - flag at all: the generated config would be selected on a different rule - than the one the operator asked for. + calls ingest_event itself -- so both halves of the classification block + (the tracker kwarg and the per-record timestamp) have to be wired up + explicitly. A classification block that reaches the first-pass trackers + but not the combo trackers is worse than none at all: the generated + config would be selected on a different rule than the one the operator + asked for. """ @staticmethod - def _records(segmentation="time"): + def _records(classification=None): detector = NewValueComboDetector( config=NewValueComboDetectorConfig( - stability_segmentation=segmentation, - timestamp_variable="ts", + auto_config_params=ComboAutoConfigParams( + classification=classification or ClassificationMethods( + index=False, time=True + ), + timestamp_variable="ts", + ), ), name="NewValueComboDetector", ) @@ -665,16 +674,59 @@ def test_combo_trackers_record_timestamps(self): combo_trackers = detector.auto_conf_persistency_combos.get_events_data()[1].get_data() assert ("var_0", "var_1") in combo_trackers tracker = combo_trackers[("var_0", "var_1")] - assert tracker.segmentation == "time" + assert tracker.classification.enabled == ("time",) assert len(tracker.timestamps) == len(tracker.change_series) == 12 assert tracker.timestamps[1] - tracker.timestamps[0] == 60.0 - def test_combo_trackers_stay_count_based_when_flag_is_off(self): - detector = self._records(segmentation="count") + def test_combo_trackers_stay_index_based_when_no_time_method_is_on(self): + detector = self._records(classification=ClassificationMethods(index=True)) detector.set_configuration(max_combo_size=2) tracker = detector.auto_conf_persistency_combos.get_events_data()[1].get_data()[ ("var_0", "var_1") ] - assert tracker.segmentation == "count" + assert tracker.classification.enabled == ("index",) assert tracker.timestamps == [] + + def test_auto_config_params_round_trip(self): + """A populated block survives from_dict -> to_dict unchanged and never + leaks into params.""" + block = { + "use_stable_vars": True, + "use_static_vars": True, + "classification": {"index": True, "time": True, "slope_index": True}, + "timestamp_variable": "level", + "timestamp_format": "%y%m%d %H%M%S", + } + source = { + "detectors": { + "NewValueComboDetector": { + "method_type": "new_value_combo_detector", + "auto_config": True, + "auto_config_params": block, + # Explicit (rather than omitted) so the first from_dict + # already coerces `events` to EventsConfig -- otherwise it + # stays the bare-dict field default and the round-trip + # equality below fails on that unrelated field, not on + # auto_config_params (pydantic does not validate/coerce + # field defaults, only explicit constructor input). + "events": {}, + } + } + } + config = NewValueComboDetectorConfig.from_dict(source, "NewValueComboDetector") + dumped = config.to_dict(method_id="NewValueComboDetector") + entry = dumped["detectors"]["NewValueComboDetector"] + auto_params = entry["auto_config_params"] + # Subset, not equality: later tasks add fields to this model and an + # exact-match assertion would break every time one lands. `classification` + # round-trips as a full six-key dict (slope_time, slope_threshold and + # decision included), so it is compared via model equality rather than + # raw dict equality. + non_classification = {k: v for k, v in block.items() if k != "classification"} + assert non_classification.items() <= auto_params.items() + assert ClassificationMethods(**auto_params["classification"]) == ClassificationMethods( + **block["classification"] + ) + assert not set(block) & set(entry.get("params", {})) + assert NewValueComboDetectorConfig.from_dict(dumped, "NewValueComboDetector") == config diff --git a/tests/test_detectors/test_persist_integration.py b/tests/test_detectors/test_persist_integration.py index f64d064b..72fa31c5 100644 --- a/tests/test_detectors/test_persist_integration.py +++ b/tests/test_detectors/test_persist_integration.py @@ -1,7 +1,14 @@ +import logging import threading import fsspec +import numpy as np +import pytest +from detectmatelibrary import schemas +from detectmatelibrary.detectors.ecvc_detector import ECVCDetector, ECVCDetectorConfig +from detectmatelibrary.detectors.scvs_detector import SCVSDetector, SCVSDetectorConfig +from detectmatelibrary.utils.sequence_encoding import decode_count_vec, encode_count_vec from detectmatelibrary.detectors.new_value_detector import NewValueDetector, NewValueDetectorConfig from detectmatelibrary.detectors.new_value_combo_detector import ( NewValueComboDetector, @@ -267,3 +274,171 @@ def ingest_loop(): stop.set() t.join(timeout=2.0) det.saver.stop() + + +# Count-vector detectors (SCVS / ECVC) ###################################### + +WINDOW_SIZE = 4 +# Distinct count vectors over EventIDs 0/1/4, each WINDOW_SIZE events long. +TRAIN_WINDOWS = [[0, 1, 4, 0], [1, 1, 0, 0], [4, 0, 1, 1], [0, 0, 4, 4]] +UNSEEN_WINDOW = [4, 4, 4, 4] + + +def _window(event_ids): + return [schemas.ParserSchema({"EventID": i}) for i in event_ids] + + +class TestCountVecCodec: + def test_round_trip(self): + assert decode_count_vec(encode_count_vec(10, (2, 1, 0, 0, 1))) == (10, (2, 1, 0, 0, 1)) + + def test_window_size_distinguishes_identical_vectors(self): + # The same count vector learned at another window size must not match. + assert encode_count_vec(4, (1, 1)) != encode_count_vec(8, (1, 1)) + + +class TestSCVSDetectorPersist: + def test_no_saver_by_default(self): + det = SCVSDetector() + assert det.saver is None + + def test_saver_created_when_persist_configured(self): + det = SCVSDetector( + name="SCVS1", + config=SCVSDetectorConfig( + auto_config=False, + persist=PersistConfig(path="memory://scvs_saver/state"), + ), + ) + assert det.saver is not None + det.saver.stop() + + def test_save_and_reload(self): + base_path = "memory://scvs_reload/state" + det_name = "SCVS_Reload" + + det1 = SCVSDetector( + name=det_name, + config=SCVSDetectorConfig( + auto_config=False, + window_size=WINDOW_SIZE, + persist=PersistConfig(path=base_path), + ), + ) + for window in TRAIN_WINDOWS: + det1.train(_window(window)) + assert isinstance(det1.saver, PersistencySaver) + det1.saver.save() + det1.saver.stop() + + det2 = SCVSDetector( + name=det_name, + config=SCVSDetectorConfig( + auto_config=False, + window_size=WINDOW_SIZE, + persist=PersistConfig(path=base_path, auto_load=True), + ), + ) + assert det2.get_known_count_vecs() == det1.get_known_count_vecs() + # A restored detector detects without retraining. + assert det2.detect(_window(TRAIN_WINDOWS[0]), schemas.DetectorSchema()) is False + assert det2.detect(_window(UNSEEN_WINDOW), schemas.DetectorSchema()) is True + det2.saver.stop() + + def test_import_state_warns_on_window_size_mismatch( + self, caplog: pytest.LogCaptureFixture + ) -> None: + det1 = SCVSDetector( + name="SCVS_WSSrc", + config=SCVSDetectorConfig(auto_config=False, window_size=WINDOW_SIZE), + ) + for window in TRAIN_WINDOWS: + det1.train(_window(window)) + state = det1.export_state() + + det2 = SCVSDetector( + name="SCVS_WSDst", + config=SCVSDetectorConfig(auto_config=False, window_size=WINDOW_SIZE + 2), + ) + with caplog.at_level(logging.WARNING): + det2.import_state(state) + assert any("window_size" in r.message for r in caplog.records) + + +class TestECVCDetectorPersist: + def test_no_saver_by_default(self): + det = ECVCDetector() + assert det.saver is None + + def test_saver_created_when_persist_configured(self): + det = ECVCDetector( + name="ECVC1", + config=ECVCDetectorConfig( + auto_config=False, + persist=PersistConfig(path="memory://ecvc_saver/state"), + ), + ) + assert det.saver is not None + det.saver.stop() + + def test_save_and_reload_rebuilds_model(self): + """A reloaded ECVC must derive the same matrix and threshold. + + post_train() splits train from validation by iteration order + over the learned vectors, so a restored model only equals a + freshly trained one because _derive() sorts them first. + """ + base_path = "memory://ecvc_reload/state" + det_name = "ECVC_Reload" + config_args = dict( + auto_config=False, + window_size=WINDOW_SIZE, + validation_per=0.5, + seed=0, + threshold_method="mean", + ) + + det1 = ECVCDetector( + name=det_name, + config=ECVCDetectorConfig( + persist=PersistConfig(path=base_path), **config_args + ), + ) + for window in TRAIN_WINDOWS: + det1.train(_window(window)) + det1.post_train() + assert det1.count_vecs is not None + assert isinstance(det1.saver, PersistencySaver) + det1.saver.save() + det1.saver.stop() + + det2 = ECVCDetector( + name=det_name, + config=ECVCDetectorConfig( + persist=PersistConfig(path=base_path, auto_load=True), **config_args + ), + ) + assert det2.count_vecs is not None + assert np.array_equal(det2.count_vecs, det1.count_vecs) + assert det2.threshold == det1.threshold + det2.saver.stop() + + def test_import_state_rebuilds_model(self): + config_args = dict( + auto_config=False, window_size=WINDOW_SIZE, validation_per=0.5, seed=0 + ) + det1 = ECVCDetector(name="ECVC_ImpSrc", config=ECVCDetectorConfig(**config_args)) + for window in TRAIN_WINDOWS: + det1.train(_window(window)) + det1.post_train() + state = det1.export_state() + + det2 = ECVCDetector(name="ECVC_ImpDst", config=ECVCDetectorConfig(**config_args)) + assert det2.count_vecs is None # nothing learned yet + det2.import_state(state) + assert det2.count_vecs is not None + assert np.array_equal(det2.count_vecs, det1.count_vecs) + + def test_untrained_detector_stays_silent(self): + det = ECVCDetector(name="ECVC_Empty", config=ECVCDetectorConfig(auto_config=False)) + assert det.detect(_window(UNSEEN_WINDOW), schemas.DetectorSchema()) is False diff --git a/tests/test_detectors/test_scvs_detector.py b/tests/test_detectors/test_scvs_detector.py index 4307c128..7d873805 100644 --- a/tests/test_detectors/test_scvs_detector.py +++ b/tests/test_detectors/test_scvs_detector.py @@ -1,5 +1,6 @@ -from detectmatelibrary.detectors.scvs_detector import build_count_vec, SCVSDetector, SCVSDetectorConfig +from detectmatelibrary.detectors.scvs_detector import SCVSDetector, SCVSDetectorConfig +from detectmatelibrary.utils.sequence_encoding import build_count_vec from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From from detectmatelibrary import schemas diff --git a/tests/test_parsers/test_drain.py b/tests/test_parsers/test_drain.py new file mode 100644 index 00000000..07aebb9b --- /dev/null +++ b/tests/test_parsers/test_drain.py @@ -0,0 +1,140 @@ +"""Most of the functionality is test it in DetectMatePerformance.""" +from detectmatelibrary.parsers.drain import DrainParser, _found_ratio + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.types_ import LogTemplates + +from detectmatelibrary import schemas + + +class TestDrainParser: + def test_train_process(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general R2D2!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_reset_after_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": True, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_not_reset_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": False, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["template"] == "hello there <*> kenobi" + + def test_not_ration_found(self): + tree_matcher = TreeMatcher(LogTemplates(["hello there <*> kenobi"])) + + logs = ["hello there general kenobi", "akuna matata"] + assert 0.5 == _found_ratio(logs, tree_matcher) + + logs = ["hello there general kenobi"] + assert 0. == _found_ratio(logs, tree_matcher) + + def test_no_auto_config_but_no_initialization(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "auto_config": True, + "data_use_configure": 2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + + assert parser.config.depth == 1 + assert parser.config.max_childs == 10 + assert parser.config.sim_thres == 0.2 diff --git a/tests/test_persistency/test_classification_methods.py b/tests/test_persistency/test_classification_methods.py new file mode 100644 index 00000000..9feabc84 --- /dev/null +++ b/tests/test_persistency/test_classification_methods.py @@ -0,0 +1,69 @@ +"""Tests for the classification-method selection model.""" + +import pytest +from pydantic import ValidationError + +from detectmatelibrary.utils.persistency.event_data_structures.trackers import ( + ClassificationMethods, +) + + +class TestDefaults: + def test_default_is_index_only_under_consensus(self): + """The default must reproduce the historical behaviour exactly.""" + m = ClassificationMethods() + assert (m.index, m.time, m.slope_index, m.slope_time) == (True, False, False, False) + assert m.decision == "consensus" + assert m.slope_threshold == -0.05 + + def test_enabled_lists_names_in_block_order(self): + m = ClassificationMethods(index=True, time=True, slope_index=False, slope_time=True) + assert m.enabled == ("index", "time", "slope_time") + + def test_enabled_of_a_single_method(self): + m = ClassificationMethods(index=False, slope_time=True) + assert m.enabled == ("slope_time",) + + +class TestNeedsTimestamps: + @pytest.mark.parametrize( + "kwargs, expected", + [ + ({}, False), + ({"index": False, "slope_index": True}, False), + ({"index": False, "time": True}, True), + ({"index": False, "slope_time": True}, True), + ({"time": True, "slope_time": True}, True), + ], + ) + def test_only_the_time_axis_methods_need_stamps(self, kwargs, expected): + assert ClassificationMethods(**kwargs).needs_timestamps is expected + + +class TestValidation: + def test_no_method_enabled_is_rejected(self): + """A method-less config would silently classify every surviving + variable STABLE, because INSUFFICIENT_DATA / STATIC / RANDOM are + decided before any method is consulted.""" + with pytest.raises(ValidationError, match="at least one classification method"): + ClassificationMethods(index=False) + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValidationError): + ClassificationMethods(segmentation="both") + + def test_unknown_decision_is_rejected(self): + with pytest.raises(ValidationError): + ClassificationMethods(decision="unanimous") + + @pytest.mark.parametrize("rule", ["consensus", "majority"]) + def test_both_decision_rules_are_accepted(self, rule): + assert ClassificationMethods(decision=rule).decision == rule + + +def test_round_trips_through_a_plain_dict(): + """to_state() and the config layer both move this model as a dict.""" + m = ClassificationMethods( + index=False, time=True, slope_time=True, slope_threshold=-0.2, decision="majority" + ) + assert ClassificationMethods(**m.model_dump()) == m diff --git a/tests/test_persistency/test_slope_stability.py b/tests/test_persistency/test_slope_stability.py new file mode 100644 index 00000000..36c56121 --- /dev/null +++ b/tests/test_persistency/test_slope_stability.py @@ -0,0 +1,783 @@ +"""Tests for the slope classification methods of the stability trackers. + +The slope is the change centroid: the mean position of the changes, +measured against the midpoint of the range those changes could occupy +and scaled by its half-span, so it lands in [-0.5, +0.5]. Index 0 is +excluded -- the first value is always recorded as a change, and keeping +it would drag every variable negative, a perfectly static one included. + +`slope_index` measures position on the index axis, `slope_time` on +normalized wall-clock time. With evenly spaced timestamps the two agree +exactly; that correspondence is what lets them share one threshold. +""" + +import numpy as np +import pytest +from pydantic import ValidationError + +from detectmatelibrary.detectors.charset_detector import CharsetDetector, CharsetDetectorConfig +from detectmatelibrary.common.variable_detector import VariableAutoConfigParams +from detectmatelibrary.utils.persistency.rle_list import RLEList +from detectmatelibrary.utils.persistency.event_data_structures.trackers import ( + StabilityClassifier, + SingleStabilityTracker, + EventStabilityTracker, + ClassificationMethods, +) + +THRESHOLDS = [1.1, 0.3, 0.1, 0.01] # same defaults SingleStabilityTracker uses + + +def make_classifier(**kwargs) -> StabilityClassifier: + return StabilityClassifier(segment_thresholds=THRESHOLDS, **kwargs) + + +def series(n: int, *change_ranges: range) -> list: + """A change series of length n with index 0 True plus the given ranges.""" + out = [False] * n + out[0] = True + for r in change_ranges: + for i in r: + out[i] = True + return out + + +def feed(tracker: SingleStabilityTracker, change_series) -> None: + """Drive a tracker so its change_series matches: fresh value on each + True, a repeat otherwise.""" + seen = 0 + for changed in change_series: + if changed: + seen += 1 + tracker.add_value(f"v{seen}") + + +# The flag only bites where the segment thresholds do not already imply an +# early centroid, and that gap opens up with series length. Over 400 samples +# the quarters are wide enough to hold 29 changes under threshold 0.3 and 9 +# under 0.1, so a variable can pass every segment test with its changes still +# sitting *late*: quarter means [0.01, 0.29, 0.09, 0.0] -> STABLE, centroid +# +0.028 -> not declining. +LATE_BUT_PASSING = series(400, range(171, 200), range(291, 300)) + +# Same length, changes up front: STABLE under the segment thresholds and +# strongly declining (centroid -0.464), so the flag leaves it alone. +EARLY = series(400, range(1, 31)) + +# Segment-UNSTABLE (quarter 1 mean 0.51 > 0.3) but strongly declining. The +# flag is a conjunct, so it can never rescue this one. +DENSE_EARLY = series(400, range(1, 151)) + + +class TestSlopeIndexAxis: + def test_hand_checked_values(self): + clf = make_classifier() + # n=5, changes at 1 and 2 -> p_bar 1.5, midpoint 2.5, half-span 3 + assert clf.slope(RLEList([True, True, True, False, False])) == -1 / 3 + # mirror image, changes at 3 and 4 -> p_bar 3.5 + assert clf.slope(RLEList([True, False, False, True, True])) == 1 / 3 + + def test_no_changes_after_the_first_hits_the_floor(self): + assert make_classifier().slope(RLEList([True] + [False] * 39)) == -0.5 + + def test_changing_every_step_is_perfectly_uniform(self): + assert make_classifier().slope(RLEList([True] * 40)) == 0.0 + + def test_too_short_to_have_a_span(self): + clf = make_classifier() + assert clf.slope(RLEList([True, False])) == 0.0 + assert clf.slope(RLEList([])) == 0.0 + + def test_stays_within_bounds(self): + clf, rng = make_classifier(), np.random.default_rng(11) + for _ in range(200): + n = int(rng.integers(3, 300)) + f = [True] + list(rng.random(n - 1) < rng.random()) + assert -0.5 <= clf.slope(RLEList(f)) <= 0.5 + + def test_rle_and_plain_list_agree(self): + clf, rng = make_classifier(), np.random.default_rng(12) + for _ in range(200): + n = int(rng.integers(3, 300)) + f = [True] + list(bool(v) for v in rng.random(n - 1) < rng.random()) + assert clf.slope(RLEList(f)) == clf.slope(f) + + def test_sign_always_matches_the_least_squares_slope(self): + """k_OLS = k * 12m / n(n-1), a strictly positive factor -- so a polyfit + over the same series can never disagree on the verdict.""" + clf, rng = make_classifier(), np.random.default_rng(13) + for _ in range(200): + n = int(rng.integers(4, 300)) + f = [True] + list(bool(v) for v in rng.random(n - 1) < rng.random()) + if not any(f[1:]): + continue + slope = np.polyfit(np.arange(1, n), np.asarray(f[1:], dtype=float), 1)[0] + assert np.sign(round(clf.slope(RLEList(f)), 12)) == np.sign(round(slope, 12)) + + +class TestSlopeTimeAxis: + def test_evenly_spaced_stamps_reproduce_the_index_axis(self): + """The property that lets both slope methods share one threshold.""" + clf, rng = make_classifier(), np.random.default_rng(14) + for _ in range(100): + n = int(rng.integers(3, 200)) + f = [True] + list(bool(v) for v in rng.random(n - 1) < rng.random()) + stamps = [float(i) for i in range(n)] + assert clf.slope(RLEList(f), stamps) == pytest.approx(clf.slope(RLEList(f))) + + def test_hand_checked_value_on_a_stretched_span(self): + """Changes at indices 1 and 2 of five, with the tail an eternity later. + + u = 1/101, 2/101 -> u_bar 0.0148515; u_first = 1/101 = 0.0099010 + k = (0.0148515 - (0.0099010 + 1) / 2) / (1 - 0.0099010) = -0.495 + The same series on evenly spaced stamps gives -1/3. + """ + clf = make_classifier() + f = RLEList([True, True, True, False, False]) + assert clf.slope(f, [0.0, 1.0, 2.0, 100.0, 101.0]) == pytest.approx(-0.495) + assert clf.slope(f, [0.0, 1.0, 2.0, 3.0, 4.0]) == pytest.approx(-1 / 3) + + def test_axes_can_disagree_in_sign(self): + """The case that motivates having both slope methods. + + [T,F,F,F,F,F,F,T,T,F] with the whole tail one long silence: the + changes sit late in *record count* (index +0.313) but they all + happened in the first moments of a long observation window (time + -0.493). Only the time axis sees that the variable settled. + """ + clf = make_classifier() + f = RLEList([True] + [False] * 6 + [True, True, False]) + stamps = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 1000.0] + assert clf.slope(f) == pytest.approx(0.3125) + assert clf.slope(f, stamps) == pytest.approx(-0.4935, abs=1e-4) + + def test_changes_late_in_the_span_score_positive(self): + clf = make_classifier() + f = RLEList([True, False, False, True, True]) + assert clf.slope(f, [0.0, 1.0, 2.0, 100.0, 101.0]) > 0.4 + + def test_no_changes_after_the_first_hits_the_floor(self): + clf = make_classifier() + stamps = [float(i) for i in range(40)] + assert clf.slope(RLEList([True] + [False] * 39), stamps) == -0.5 + + def test_stays_within_bounds(self): + clf, rng = make_classifier(), np.random.default_rng(15) + for _ in range(200): + n = int(rng.integers(3, 300)) + f = [True] + list(bool(v) for v in rng.random(n - 1) < rng.random()) + stamps = list(np.cumsum(rng.random(n))) + assert -0.5 <= clf.slope(RLEList(f), stamps) <= 0.5 + + def test_rle_and_plain_list_agree(self): + clf, rng = make_classifier(), np.random.default_rng(16) + for _ in range(100): + n = int(rng.integers(3, 200)) + f = [True] + list(bool(v) for v in rng.random(n - 1) < rng.random()) + stamps = list(np.cumsum(rng.random(n))) + assert clf.slope(RLEList(f), stamps) == clf.slope(f, stamps) + + +class TestSlopeTimeFallsBackToIndex: + """Time-aware classification is best-effort: it degrades to the index + axis rather than failing a run or passing unconditionally.""" + + SERIES = RLEList([True, True, True, False, False]) + + def index_value(self): + return make_classifier().slope(self.SERIES) + + def test_no_timestamps(self): + assert make_classifier().slope(self.SERIES, None) == self.index_value() + + def test_length_mismatch(self): + assert make_classifier().slope(self.SERIES, [0.0, 1.0]) == self.index_value() + + def test_zero_span(self): + stamps = [7.0] * 5 + assert make_classifier().slope(self.SERIES, stamps) == self.index_value() + + def test_out_of_order(self): + stamps = [0.0, 5.0, 2.0, 6.0, 7.0] + assert make_classifier().slope(self.SERIES, stamps) == self.index_value() + + def test_non_finite_entry(self): + stamps = [0.0, 1.0, float("nan"), 3.0, 4.0] + assert make_classifier().slope(self.SERIES, stamps) == self.index_value() + + def test_none_entry(self): + stamps = [0.0, 1.0, None, 3.0, 4.0] + assert make_classifier().slope(self.SERIES, stamps) == self.index_value() + + def test_zero_achievable_range(self): + """t_1 == t_last: every countable position shares one instant, so the + time axis has no range to normalize against.""" + stamps = [0.0, 9.0, 9.0, 9.0, 9.0] + assert make_classifier().slope(self.SERIES, stamps) == self.index_value() + + +class TestSlopeReportsItsAxis: + def test_time_axis_when_usable(self): + clf = make_classifier() + _, axis = clf._slope(RLEList([True, True, False, False]), [0.0, 1.0, 2.0, 3.0]) + assert axis == "time" + + def test_index_axis_on_fallback(self): + clf = make_classifier() + _, axis = clf._slope(RLEList([True, True, False, False]), None) + assert axis == "index" + + +class TestSlopeVerdictsOnTrackers: + def test_off_by_default_changes_nothing(self): + tracker = SingleStabilityTracker() + feed(tracker, LATE_BUT_PASSING) + assert tracker.classification.enabled == ("index",) + assert tracker.classify().type == "STABLE" + + def test_slope_index_flips_a_late_but_passing_variable(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + feed(tracker, LATE_BUT_PASSING) + assert tracker.classify().type == "UNSTABLE" + + def test_slope_index_leaves_an_early_variable_alone(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + feed(tracker, EARLY) + assert tracker.classify().type == "STABLE" + + def test_consensus_can_only_tighten_never_loosen(self): + off = SingleStabilityTracker() + on = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + feed(off, DENSE_EARLY) + feed(on, DENSE_EARLY) + # strongly declining (-0.313) but index-UNSTABLE -> stays UNSTABLE + assert off.classify().type == "UNSTABLE" + assert on.classify().type == "UNSTABLE" + + def test_slope_index_can_stand_alone(self): + """DENSE_EARLY is index-UNSTABLE but strongly declining. + + With the segment methods off, only the centroid decides -- and + it says stable. + """ + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=False, slope_index=True) + ) + feed(tracker, DENSE_EARLY) + assert tracker.classify().type == "STABLE" + + def test_threshold_is_configurable(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + feed(tracker, LATE_BUT_PASSING) + assert tracker.classify().type == "UNSTABLE" + # centroid is +0.028; a threshold above it lets the variable through + tracker.classification = ClassificationMethods( + index=True, slope_index=True, slope_threshold=0.1 + ) + assert tracker.classify().type == "STABLE" + + def test_reason_names_every_enabled_method_and_the_decision(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + feed(tracker, EARLY) + reason = tracker.classify().reason + assert "index:" in reason and "slope_index:" in reason + assert "decision=consensus (2/2)" in reason + + def test_reason_omits_methods_that_are_off(self): + tracker = SingleStabilityTracker() + feed(tracker, EARLY) + reason = tracker.classify().reason + assert "slope_index:" not in reason and "time:" not in reason + + def test_early_classify_reasons_are_untouched(self): + """STATIC / RANDOM / INSUFFICIENT_DATA are decided before any method is + consulted, so no method setting can reach them.""" + static = SingleStabilityTracker( + classification=ClassificationMethods(index=False, slope_time=True) + ) + for _ in range(10): + static.add_value("a", timestamp=1.0) + assert static.classify().type == "STATIC" + + short = SingleStabilityTracker() + short.add_value("a") + assert short.classify().type == "INSUFFICIENT_DATA" + + +class TestTimestampCollection: + def test_index_only_collects_nothing(self): + tracker = SingleStabilityTracker() + tracker.add_value("a", timestamp=1.0) + assert tracker.timestamps == [] + + def test_slope_index_only_collects_nothing(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=False, slope_index=True) + ) + tracker.add_value("a", timestamp=1.0) + assert tracker.timestamps == [] + + def test_slope_time_collects_stamps(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=False, slope_time=True) + ) + tracker.add_value("a", timestamp=1.0) + tracker.add_value("b", timestamp=2.0) + assert tracker.timestamps == [1.0, 2.0] + + def test_slope_time_without_stamps_falls_back_to_the_index_axis(self): + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=False, slope_time=True) + ) + feed(tracker, LATE_BUT_PASSING) # feed() passes no timestamps + assert tracker.timestamps == [] + assert "index axis" in tracker.classify().reason + + +class TestClassificationIsSwappable: + """The parent repo's notebooks get several verdicts from one ingest by + reassigning this between classify() calls.""" + + def test_reassignment_changes_the_verdict(self): + tracker = SingleStabilityTracker() + feed(tracker, LATE_BUT_PASSING) + assert tracker.classify().type == "STABLE" + tracker.classification = ClassificationMethods(index=True, slope_index=True) + assert tracker.classify().type == "UNSTABLE" + + def test_the_classifier_cannot_drift_from_the_tracker(self): + """One property over one owner -- not two attributes to keep in + sync.""" + tracker = SingleStabilityTracker() + tracker.classification = ClassificationMethods(index=False, time=True) + assert tracker.stability_classifier.classification is tracker.classification + assert tracker.classification.needs_timestamps is True + + def test_accepts_a_plain_dict(self): + """State and config both deliver the block as a dict.""" + tracker = SingleStabilityTracker(classification={"index": True, "time": True}) + assert tracker.classification == ClassificationMethods(index=True, time=True) + + def test_setter_accepts_a_plain_dict(self): + """The setter coerces too, not just the constructor.""" + tracker = SingleStabilityTracker() + tracker.classification = {"index": True, "slope_index": True, "slope_threshold": 0.2} + assert tracker.stability_classifier.classification == ClassificationMethods( + index=True, slope_index=True, slope_threshold=0.2 + ) + assert tracker.classification == ClassificationMethods( + index=True, slope_index=True, slope_threshold=0.2 + ) + + +class TestStatePersistence: + def test_round_trip_preserves_the_block(self): + tracker = SingleStabilityTracker(classification=ClassificationMethods( + index=True, slope_index=True, slope_threshold=-0.2, decision="majority", + )) + feed(tracker, EARLY) + restored = SingleStabilityTracker.from_state(tracker.to_state()) + assert restored.classification == tracker.classification + assert restored.classify().type == tracker.classify().type + + def test_state_is_msgpack_plain(self): + """to_state() must be msgpack-compatible: a pydantic model is not.""" + state = SingleStabilityTracker().to_state() + assert isinstance(state["classification"], dict) + assert set(state["classification"]) == { + "index", "time", "slope_index", "slope_time", "slope_threshold", "decision", + } + + def test_old_keys_are_gone_from_state(self): + state = SingleStabilityTracker().to_state() + for key in ("segmentation", "require_declining", "incline_threshold"): + assert key not in state + + def test_the_note_is_not_persisted(self): + tracker = SingleStabilityTracker() + feed(tracker, EARLY) + tracker.classify() + assert "_stability_note" not in tracker.to_state() + + def test_event_tracker_propagates_the_block(self): + event_tracker = EventStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + event_tracker.add_data({"var1": "a"}) + event_tracker.add_data({"var1": "b"}) + assert event_tracker.get_data()["var1"].classification.enabled == ( + "index", "slope_index", + ) + + def test_event_tracker_dump_load_preserves_the_block(self): + event_tracker = EventStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + event_tracker.add_data({"var1": "a"}) + event_tracker.add_data({"var1": "b"}) + restored = EventStabilityTracker.load( + event_tracker.dump(), + classification={"index": True, "slope_index": True}, + ) + assert restored.get_data()["var1"].classification.enabled == ( + "index", "slope_index", + ) + + +class TestLegacyStateMigration: + """Snapshots written before this change must keep loading. + + This is the one place the old names survive. + """ + + def legacy_state(self, **overrides): + state = SingleStabilityTracker().to_state() + del state["classification"] + state.update(overrides) + return state + + def test_no_stability_keys_at_all(self): + restored = SingleStabilityTracker.from_state(self.legacy_state()) + assert restored.classification == ClassificationMethods() + + def test_segmentation_count(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state(segmentation="count") + ) + assert restored.classification.enabled == ("index",) + + def test_segmentation_time_means_time_alone(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state(segmentation="time") + ) + assert restored.classification.enabled == ("time",) + + def test_segmentation_both(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state(segmentation="both") + ) + assert restored.classification.enabled == ("index", "time") + + def test_require_declining_becomes_slope_index(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state(segmentation="count", require_declining=True) + ) + assert restored.classification.enabled == ("index", "slope_index") + + def test_incline_threshold_becomes_slope_threshold(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state( + segmentation="count", require_declining=True, incline_threshold=-0.25 + ) + ) + assert restored.classification.slope_threshold == -0.25 + + def test_legacy_states_always_decide_by_consensus(self): + restored = SingleStabilityTracker.from_state( + self.legacy_state(segmentation="both", require_declining=True) + ) + assert restored.classification.decision == "consensus" + assert restored.classification.enabled == ("index", "time", "slope_index") + + def test_legacy_state_without_add_value_keys_still_loads(self): + """Old enough to predate add_value_fn as well.""" + state = self.legacy_state(segmentation="both") + del state["add_value_fn"], state["detector_config"] + restored = SingleStabilityTracker.from_state(state) + assert restored.classification.enabled == ("index", "time") + + def test_fed_tracker_with_deleted_classification_key_classifies_correctly(self): + """End-to-end: legacy snapshot with real observations restores and + classifies correctly. This verifies the migration works not just for + config translation but for the full state round-trip.""" + tracker = SingleStabilityTracker() + feed(tracker, EARLY) + state = tracker.to_state() + del state["classification"] + restored = SingleStabilityTracker.from_state(state) + assert restored.classify().type == "STABLE" + + +class TestConfigWiring: + def test_block_reaches_per_variable_trackers(self): + # CharsetDetector's `config` default is a shared mutable instance, so + # pass explicit fresh configs (see test_time_dependent_stability.py). + # + # classification only shapes the configure-phase persistency: the + # trained persistency is read by _check_variable, which never calls + # classify(), so it never receives classification kwargs at all. + default = CharsetDetector(config=CharsetDetectorConfig()) + assert default.persistency.event_data_kwargs.get("classification") is None + + configured = CharsetDetector(config=CharsetDetectorConfig()) + configured.config.auto_config_params.classification = ClassificationMethods( + index=True, slope_index=True + ) + rebuilt = CharsetDetector(config=configured.config.to_dict(method_id="CharsetDetector")) + block = rebuilt.auto_conf_persistency.event_data_kwargs["classification"] + assert ClassificationMethods(**block).enabled == ("index", "slope_index") + + def test_default_block_is_not_forwarded(self): + """Forwarding the default would be noise; the tracker already has + it.""" + default = CharsetDetector(config=CharsetDetectorConfig()) + assert "classification" not in (default.auto_conf_persistency.event_data_kwargs or {}) + + def test_config_field_round_trips(self): + detector = CharsetDetector(config=CharsetDetectorConfig()) + detector.config.auto_config_params.classification = ClassificationMethods( + index=False, slope_time=True, decision="majority" + ) + restored = type(detector.config).from_dict( + detector.config.to_dict(method_id="CharsetDetector"), "CharsetDetector" + ) + assert restored.auto_config_params.classification.enabled == ("slope_time",) + assert restored.auto_config_params.classification.decision == "majority" + + def test_survives_auto_config(self): + """set_configuration() writes only config.events and flips + config.auto_config to False -- it never touches auto_config_params, so + operator settings survive because nothing overwrites them.""" + detector = CharsetDetector(config=CharsetDetectorConfig( + auto_config=True, + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods(index=True, slope_index=True), + use_static_vars=False, + ), + )) + detector.set_configuration() + assert detector.config.auto_config_params.classification.enabled == ( + "index", "slope_index", + ) + assert detector.config.auto_config_params.use_static_vars is False + + def test_index_axis_methods_pull_in_no_timestamp_requirement(self): + detector = CharsetDetector(config=CharsetDetectorConfig()) + detector.config.auto_config_params.classification = ClassificationMethods( + index=True, slope_index=True + ) + rebuilt = CharsetDetector(config=detector.config.to_dict(method_id="CharsetDetector")) + assert "classification" not in (rebuilt.persistency.event_data_kwargs or {}) + + tracker = SingleStabilityTracker( + classification=ClassificationMethods(index=True, slope_index=True) + ) + tracker.add_value("a", timestamp=1.0) + assert tracker.timestamps == [] + + +class TestOldConfigFieldsAreRejected: + """Clean break: AutoConfigParams sets extra='forbid', so the old + spellings raise instead of being silently ignored.""" + + @pytest.mark.parametrize( + "kwargs", + [ + {"segmentation": "both"}, + {"require_declining": True}, + {"incline_threshold": -0.25}, + ], + ) + def test_rejected(self, kwargs): + with pytest.raises(ValidationError): + VariableAutoConfigParams(**kwargs) + + +def test_slope_threshold_reaches_the_classifier(): + """The threshold is configuration, not a constant buried in the + classifier.""" + from detectmatelibrary.detectors.new_value_detector import ( + NewValueDetector, + NewValueDetectorConfig, + ) + + detector = NewValueDetector( + name="NewValueDetector", + config=NewValueDetectorConfig( + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods( + index=True, slope_index=True, slope_threshold=-0.25 + ), + ), + ), + ) + persistency = detector.auto_conf_persistency + tracker = persistency.event_data_class(**persistency.event_data_kwargs) + single = tracker.single_tracker_type() + assert single.classification.enabled == ("index", "slope_index") + assert single.stability_classifier.classification.slope_threshold == -0.25 + + +# Fixture reused from test_time_dependent_stability.py: 30 fresh values one +# second apart, then the same value repeated 10 times spread over ~17 minutes. +# index -> means [1.0, 1.0, 1.0, 0.0] -> UNSTABLE +# time -> means [0.938, 0.0, 0.0, 0.0] -> STABLE +# slopes -> -0.132 (index), -0.486 (time) -> both STABLE +# The one fixture that splits 3-1, which is what makes majority testable at +# four enabled methods. +CHURN_SERIES = [True] * 30 + [False] * 10 +CHURN_TIMES = [float(i) for i in range(30)] + [100.0 * (i + 1) for i in range(10)] + +UNIFORM_400 = [float(i) for i in range(400)] + + +def methods(**kwargs) -> ClassificationMethods: + """A method block with every default overridable, index included.""" + return ClassificationMethods(**{"index": False, **kwargs}) + + +class TestVerdicts: + def test_only_enabled_methods_appear_in_block_order(self): + clf = make_classifier(classification=methods(slope_time=True, index=True)) + assert list(clf.verdicts(RLEList(LATE_BUT_PASSING), UNIFORM_400)) == [ + "index", "slope_time", + ] + + def test_late_but_passing_splits_segments_from_slope(self): + clf = make_classifier( + classification=ClassificationMethods(index=True, slope_index=True) + ) + assert clf.verdicts(RLEList(LATE_BUT_PASSING)) == { + "index": True, "slope_index": False, + } + + def test_churn_fixture_splits_three_to_one(self): + clf = make_classifier(classification=ClassificationMethods( + index=True, time=True, slope_index=True, slope_time=True, + )) + assert clf.verdicts(RLEList(CHURN_SERIES), CHURN_TIMES) == { + "index": False, "time": True, "slope_index": True, "slope_time": True, + } + + def test_a_slope_method_can_stand_alone(self): + clf = make_classifier(classification=methods(slope_index=True)) + assert clf.verdicts(RLEList(EARLY)) == {"slope_index": True} + + def test_standing_alone_skips_the_segment_means(self): + """With no segment-threshold method enabled the means are never + computed, so the classifier must not report stale ones.""" + clf = make_classifier(classification=methods(slope_index=True)) + clf.verdicts(RLEList(EARLY)) + assert clf.get_last_segment_means() == [] + + def test_empty_series_is_stable_under_every_method(self): + clf = make_classifier(classification=ClassificationMethods( + index=True, time=True, slope_index=True, slope_time=True, + )) + assert clf.verdicts(RLEList([])) == { + "index": True, "time": True, "slope_index": True, "slope_time": True, + } + + def test_slope_threshold_is_read_from_the_block(self): + late = RLEList(LATE_BUT_PASSING) # centroid +0.028 + strict = make_classifier(classification=methods(slope_index=True)) + assert strict.verdicts(late) == {"slope_index": False} + loose = make_classifier( + classification=methods(slope_index=True, slope_threshold=0.1) + ) + assert loose.verdicts(late) == {"slope_index": True} + + +class TestDecisionRule: + """Consensus and majority agree at one and two enabled methods and diverge + at three and four. + + Ties resolve to UNSTABLE. + """ + + @pytest.mark.parametrize( + "verdicts, consensus, majority", + [ + ({"a": True}, True, True), + ({"a": False}, False, False), + ({"a": True, "b": True}, True, True), + ({"a": True, "b": False}, False, False), # 1-1 tie + ({"a": True, "b": True, "c": True}, True, True), + ({"a": True, "b": True, "c": False}, False, True), # 2/3 + ({"a": True, "b": False, "c": False}, False, False), + ({"a": True, "b": True, "c": True, "d": True}, True, True), + ({"a": True, "b": True, "c": True, "d": False}, False, True), # 3/4 + ({"a": True, "b": True, "c": False, "d": False}, False, False), # 2-2 tie + ], + ) + def test_table(self, verdicts, consensus, majority): + for rule, expected in (("consensus", consensus), ("majority", majority)): + clf = make_classifier( + classification=ClassificationMethods(decision=rule) + ) + assert clf.decide(verdicts) is expected + + def test_majority_rescues_the_three_to_one_fixture(self): + block = dict(index=True, time=True, slope_index=True, slope_time=True) + strict = make_classifier( + classification=ClassificationMethods(**block, decision="consensus") + ) + lenient = make_classifier( + classification=ClassificationMethods(**block, decision="majority") + ) + assert strict.is_stable(RLEList(CHURN_SERIES), CHURN_TIMES) is False + assert lenient.is_stable(RLEList(CHURN_SERIES), CHURN_TIMES) is True + + def test_two_two_tie_stays_unstable(self): + block = dict(index=True, time=True, slope_index=True, slope_time=True) + lenient = make_classifier( + classification=ClassificationMethods(**block, decision="majority") + ) + # index/time STABLE, both slopes UNSTABLE (centroid +0.028) + assert lenient.is_stable(RLEList(LATE_BUT_PASSING), UNIFORM_400) is False + + +class TestFallbackDoubleCount: + """A fallen-back method still casts its vote. + + Dropping it instead would make the enabled count vary per variable, + so majority would mean something different for each one. + """ + + def test_both_slopes_vote_the_same_way_without_stamps(self): + clf = make_classifier( + classification=methods(slope_index=True, slope_time=True) + ) + assert clf.verdicts(RLEList(LATE_BUT_PASSING), None) == { + "slope_index": False, "slope_time": False, + } + + def test_the_details_name_the_axis_actually_used(self): + clf = make_classifier( + classification=methods(slope_index=True, slope_time=True) + ) + clf.verdicts(RLEList(LATE_BUT_PASSING), None) + assert "index axis" in clf.get_last_details()["slope_time"] + + +class TestDetails: + def test_segment_method_reports_means_and_thresholds(self): + clf = make_classifier(classification=ClassificationMethods(index=True)) + clf.verdicts(RLEList(EARLY)) + detail = clf.get_last_details()["index"] + assert "index:" in detail and "STABLE" in detail + assert str(THRESHOLDS) in detail + + def test_slope_method_reports_the_centroid_and_threshold(self): + clf = make_classifier(classification=methods(slope_index=True)) + clf.verdicts(RLEList(LATE_BUT_PASSING)) + detail = clf.get_last_details()["slope_index"] + assert "slope_index:" in detail and "UNSTABLE" in detail + assert "-0.05" in detail and "index axis" in detail + + def test_details_cover_exactly_the_enabled_methods(self): + clf = make_classifier(classification=ClassificationMethods( + index=True, slope_time=True, + )) + verdicts = clf.verdicts(RLEList(EARLY), UNIFORM_400) + assert set(clf.get_last_details()) == set(verdicts) diff --git a/tests/test_persistency/test_time_dependent_stability.py b/tests/test_persistency/test_time_dependent_stability.py index f8e03d88..95ac392e 100644 --- a/tests/test_persistency/test_time_dependent_stability.py +++ b/tests/test_persistency/test_time_dependent_stability.py @@ -1,23 +1,37 @@ -"""Tests for the stability_segmentation option of the stability trackers.""" +"""Tests for the time-axis classification methods of the stability trackers.""" import logging import math import detectmatelibrary.schemas as schemas from detectmatelibrary.detectors.charset_detector import CharsetDetector, CharsetDetectorConfig +from detectmatelibrary.common.variable_detector import VariableAutoConfigParams from detectmatelibrary.utils.persistency.rle_list import RLEList from detectmatelibrary.utils.persistency import EventPersistency from detectmatelibrary.utils.persistency.event_data_structures.trackers import ( StabilityClassifier, SingleStabilityTracker, EventStabilityTracker, + ClassificationMethods, ) THRESHOLDS = [1.1, 0.3, 0.1, 0.01] # same defaults SingleStabilityTracker uses -def make_classifier() -> StabilityClassifier: - return StabilityClassifier(segment_thresholds=THRESHOLDS) +def make_index_classifier() -> StabilityClassifier: + """Equal-index cuts -- the reference every fallback test compares to.""" + return StabilityClassifier( + segment_thresholds=THRESHOLDS, + classification=ClassificationMethods(index=True), + ) + + +def make_time_classifier() -> StabilityClassifier: + """Equal-duration cuts -- what this file is about.""" + return StabilityClassifier( + segment_thresholds=THRESHOLDS, + classification=ClassificationMethods(index=False, time=True), + ) # Divergence fixture: 3 changes up front, then a quiet tail of 37. @@ -59,60 +73,60 @@ def make_classifier() -> StabilityClassifier: class TestClassifierTimeBoundaries: - def test_count_mode_is_stable_on_divergent_fixture(self): - clf = make_classifier() + def test_index_mode_is_stable_on_divergent_fixture(self): + clf = make_index_classifier() # count segments of 10: means [0.3, 0, 0, 0] -> all below thresholds assert clf.is_stable(RLEList(DIVERGENT_SERIES)) is True def test_time_mode_is_unstable_on_divergent_fixture(self): - clf = make_classifier() + clf = make_time_classifier() # time quarters put a lone change (mean 1.0) into segment 2 (thresh 0.3) assert clf.is_stable(RLEList(DIVERGENT_SERIES), timestamps=DIVERGENT_TIMES) is False - def test_uniform_timestamps_match_count_mode(self): + def test_uniform_timestamps_match_index_mode(self): # N divisible by n_segments -> boundaries coincide exactly series = [True, False, False, True, False, False, False, False] ts = [float(i) for i in range(8)] - clf_count = make_classifier() + clf_count = make_index_classifier() clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() clf_time.is_stable(RLEList(series), timestamps=ts) assert clf_time.get_last_segment_means() == clf_count.get_last_segment_means() def test_plain_list_path_supports_timestamps(self): - clf = make_classifier() + clf = make_time_classifier() assert clf.is_stable(list(DIVERGENT_SERIES), timestamps=DIVERGENT_TIMES) is False - def test_zero_span_falls_back_to_count_mode(self): + def test_zero_span_falls_back_to_index_mode(self): series = [True, False, False, False, False, False, False, False] - clf_count = make_classifier() + clf_count = make_index_classifier() expected = clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() result = clf_time.is_stable(RLEList(series), timestamps=[5.0] * 8) assert result == expected assert clf_time.get_last_segment_means() == clf_count.get_last_segment_means() - def test_length_mismatch_falls_back_to_count_mode(self): + def test_length_mismatch_falls_back_to_index_mode(self): series = [True, False, False, False, False, False, False, False] - clf_count = make_classifier() + clf_count = make_index_classifier() expected = clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() assert clf_time.is_stable(RLEList(series), timestamps=[1.0, 2.0]) == expected - def test_none_timestamp_entry_falls_back_to_count_mode(self): + def test_none_timestamp_entry_falls_back_to_index_mode(self): series = [True, False, False, False, False, False, False, False] - clf_count = make_classifier() + clf_count = make_index_classifier() expected = clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() ts = [0.0, 1.0, None, 3.0, 4.0, 5.0, 6.0, 7.0] assert clf_time.is_stable(RLEList(series), timestamps=ts) == expected assert clf_time.get_last_segment_means() == clf_count.get_last_segment_means() - def test_nan_timestamp_entry_falls_back_to_count_mode(self): + def test_nan_timestamp_entry_falls_back_to_index_mode(self): series = [True, False, False, False, False, False, False, False] - clf_count = make_classifier() + clf_count = make_index_classifier() expected = clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() ts = [0.0, 1.0, float("nan"), 3.0, 4.0, 5.0, 6.0, 7.0] assert clf_time.is_stable(RLEList(series), timestamps=ts) == expected assert clf_time.get_last_segment_means() == clf_count.get_last_segment_means() @@ -126,19 +140,19 @@ def test_empty_time_segment_scores_zero(self): alone is lenient on a burst followed by silence. Count mode still sees the churn, so `both` catches it. """ - clf_count = make_classifier() + clf_count = make_index_classifier() assert clf_count.is_stable(RLEList(BURSTY_SERIES)) is False - clf_time = make_classifier() + clf_time = make_time_classifier() assert clf_time.is_stable(RLEList(BURSTY_SERIES), timestamps=BURSTY_TIMES) is True assert clf_time.get_last_segment_means() == [0.5, 0.0, 0.0, 0.0] def test_empty_time_segment_scores_zero_on_plain_list_path(self): - clf_time = make_classifier() + clf_time = make_time_classifier() assert clf_time.is_stable(list(BURSTY_SERIES), timestamps=BURSTY_TIMES) is True assert clf_time.get_last_segment_means() == [0.5, 0.0, 0.0, 0.0] - def test_out_of_order_timestamps_fall_back_to_count_mode(self): + def test_out_of_order_timestamps_fall_back_to_index_mode(self): """np.searchsorted requires sorted input. UNSORTED_TIMES is SORTED_TIMES with two entries transposed -- @@ -152,35 +166,35 @@ def test_out_of_order_timestamps_fall_back_to_count_mode(self): unsorted_times = list(sorted_times) unsorted_times[9], unsorted_times[30] = unsorted_times[30], unsorted_times[9] - clf_count = make_classifier() + clf_count = make_index_classifier() expected = clf_count.is_stable(RLEList(series)) - clf_time = make_classifier() + clf_time = make_time_classifier() assert clf_time.is_stable(RLEList(series), timestamps=unsorted_times) == expected assert clf_time.get_last_segment_means() == clf_count.get_last_segment_means() def test_sorted_timestamps_still_use_time_mode(self): """The monotonicity guard must not disable time mode for valid input.""" - clf = make_classifier() + clf = make_time_classifier() assert clf.is_stable(RLEList(DIVERGENT_SERIES), timestamps=DIVERGENT_TIMES) is False - count = make_classifier() - count.is_stable(RLEList(DIVERGENT_SERIES)) - assert clf.get_last_segment_means() != count.get_last_segment_means() + index = make_index_classifier() + index.is_stable(RLEList(DIVERGENT_SERIES)) + assert clf.get_last_segment_means() != index.get_last_segment_means() def test_equal_timestamps_are_not_treated_as_out_of_order(self): """Duplicate stamps are non-decreasing, so they stay in time mode.""" series = [True, False] * 20 times = [float(i // 2) for i in range(40)] # each stamp used twice - clf = make_classifier() + clf = make_time_classifier() assert clf.is_stable(RLEList(series), timestamps=times) is False assert not any(math.isnan(mean) for mean in clf.get_last_segment_means()) def test_list_and_rle_agree_on_ragged_length(self): """13 items over 4 segments: both paths must cut identically.""" series = [True, False, True] + [False] * 10 - clf_list = make_classifier() + clf_list = make_index_classifier() clf_list.is_stable(list(series)) - clf_rle = make_classifier() + clf_rle = make_index_classifier() clf_rle.is_stable(RLEList(series)) assert clf_list.get_last_segment_means() == clf_rle.get_last_segment_means() @@ -207,12 +221,12 @@ def feed_agreeing(tracker: SingleStabilityTracker) -> None: class TestSingleStabilityTrackerSegmentation: def test_timestamps_stored_only_when_enabled(self): - on = SingleStabilityTracker(segmentation="time") + on = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) on.add_value("a", timestamp=1.0) on.add_value("b", timestamp=2.0) assert on.timestamps == [1.0, 2.0] - off = SingleStabilityTracker() # default segmentation="count" + off = SingleStabilityTracker() # default classification=ClassificationMethods(index=True) off.add_value("a", timestamp=1.0) assert off.timestamps == [] @@ -221,13 +235,13 @@ def test_classification_diverges_between_modes(self): feed_divergent(count_mode) assert count_mode.classify().type == "STABLE" - time_mode = SingleStabilityTracker(segmentation="time") + time_mode = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) feed_divergent(time_mode) assert time_mode.classify().type == "UNSTABLE" - def test_missing_timestamps_fall_back_to_count_mode(self): - # time segmentation on, but values arrive without timestamps - tracker = SingleStabilityTracker(segmentation="time") + def test_missing_timestamps_fall_back_to_index_mode(self): + # time classification on, but values arrive without timestamps + tracker = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) for value in ["a", "b", "c"] + ["c"] * 37: tracker.add_value(value) reference = SingleStabilityTracker() @@ -236,10 +250,10 @@ def test_missing_timestamps_fall_back_to_count_mode(self): assert tracker.classify().type == reference.classify().type def test_round_trip_preserves_time_state(self): - tracker = SingleStabilityTracker(segmentation="time") + tracker = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) feed_divergent(tracker) restored = SingleStabilityTracker.from_state(tracker.to_state()) - assert restored.segmentation == "time" + assert restored.classification == ClassificationMethods(index=False, time=True) assert restored.timestamps == tracker.timestamps assert restored.classify().type == "UNSTABLE" @@ -247,10 +261,10 @@ def test_legacy_state_without_time_keys_defaults_off(self): tracker = SingleStabilityTracker() tracker.add_value("hello") state = tracker.to_state() - state.pop("segmentation", None) # simulate pre-flag snapshot + state.pop("classification", None) # simulate pre-flag snapshot state.pop("timestamps", None) restored = SingleStabilityTracker.from_state(state) - assert restored.segmentation == "count" + assert restored.classification == ClassificationMethods(index=True) assert restored.timestamps == [] def test_legacy_state_without_add_value_keys_loads(self): @@ -268,35 +282,38 @@ def test_legacy_state_without_add_value_keys_loads(self): assert restored.detector_config is None assert restored.unique_set == {"hello"} - def test_bursty_series_needs_the_count_pass(self): + def test_bursty_series_needs_the_index_pass(self): """A burst followed by silence is time-STABLE (empty quarters score - 0.0) but count-UNSTABLE, so only `both` refuses to hand it to auto- + 0.0) but index-UNSTABLE, so only `both` refuses to hand it to auto- config variable selection as a monitoring candidate.""" trackers = { - mode: SingleStabilityTracker(segmentation=mode) - for mode in ("count", "time", "both") + "index": SingleStabilityTracker(classification=ClassificationMethods(index=True)), + "time": SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)), + "both": SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)), } for value, ts in zip(BURSTY_VALUES, BURSTY_TIMES): for tracker in trackers.values(): tracker.add_value(value, timestamp=ts) - assert trackers["count"].classify().type == "UNSTABLE" + assert trackers["index"].classify().type == "UNSTABLE" assert trackers["time"].classify().type == "STABLE" assert trackers["both"].classify().type == "UNSTABLE" class TestSegmentationPlumbing: def test_event_tracker_propagates_flag_and_timestamp(self): - event_tracker = EventStabilityTracker(segmentation="time") + event_tracker = EventStabilityTracker( + classification=ClassificationMethods(index=False, time=True) + ) event_tracker.add_data({"var1": "a"}, timestamp=1.0) event_tracker.add_data({"var1": "b"}, timestamp=2.0) single = event_tracker.get_data()["var1"] - assert single.segmentation == "time" + assert single.classification == ClassificationMethods(index=False, time=True) assert single.timestamps == [1.0, 2.0] def test_ingest_event_forwards_timestamp(self): storage = EventPersistency( EventStabilityTracker, - event_data_kwargs={"segmentation": "time"}, + event_data_kwargs={"classification": {"index": False, "time": True}}, ) storage.ingest_event(1, "tpl <*>", variables=["a"], timestamp=10.0) storage.ingest_event(1, "tpl <*>", variables=["b"], timestamp=20.0) @@ -311,12 +328,16 @@ def test_ingest_event_without_timestamp_still_works(self): assert single.timestamps == [] def test_event_tracker_dump_load_preserves_timestamps(self): - event_tracker = EventStabilityTracker(segmentation="time") + event_tracker = EventStabilityTracker( + classification=ClassificationMethods(index=False, time=True) + ) event_tracker.add_data({"var1": "a"}, timestamp=1.0) event_tracker.add_data({"var1": "b"}, timestamp=2.0) - restored = EventStabilityTracker.load(event_tracker.dump(), segmentation="time") + restored = EventStabilityTracker.load( + event_tracker.dump(), classification={"index": False, "time": True} + ) single = restored.get_data()["var1"] - assert single.segmentation == "time" + assert single.classification == ClassificationMethods(index=False, time=True) assert single.timestamps == [1.0, 2.0] @@ -326,7 +347,8 @@ class TestSegmentationWithDetectorAddValueFn: def test_detector_backed_tracker_records_timestamps(self): tracker = SingleStabilityTracker( - add_value_fn="CharsetDetector", segmentation="time" + add_value_fn="CharsetDetector", + classification=ClassificationMethods(index=False, time=True), ) tracker.add_value("ab", timestamp=1.0) tracker.add_value("cd", timestamp=2.0) @@ -338,7 +360,8 @@ def test_value_range_skipped_value_keeps_alignment(self): """ValueRangeDetector returns early on non-numeric input without appending to change_series; timestamps must not drift.""" tracker = SingleStabilityTracker( - add_value_fn="ValueRangeDetector", segmentation="time" + add_value_fn="ValueRangeDetector", + classification=ClassificationMethods(index=False, time=True), ) tracker.add_value("1", timestamp=1.0) tracker.add_value("not-a-number", timestamp=2.0) # detector records nothing @@ -348,12 +371,15 @@ def test_value_range_skipped_value_keeps_alignment(self): def test_event_tracker_detector_backed_round_trip(self): event_tracker = EventStabilityTracker( - add_value_fn="CharsetDetector", segmentation="time" + add_value_fn="CharsetDetector", + classification=ClassificationMethods(index=False, time=True), ) event_tracker.add_data({"var1": "ab"}, timestamp=1.0) event_tracker.add_data({"var1": "cd"}, timestamp=2.0) restored = EventStabilityTracker.load( - event_tracker.dump(), add_value_fn="CharsetDetector", segmentation="time" + event_tracker.dump(), + add_value_fn="CharsetDetector", + classification={"index": False, "time": True}, ) single = restored.get_data()["var1"] assert single.unique_set == {"a", "b", "c", "d"} @@ -379,7 +405,7 @@ class TestTimestampResolution: # explicitly rather than bare CharsetDetector(). CharsetDetector.__init__'s # `config` default argument is a single shared CharsetDetectorConfig() # instance (pre-existing mutable-default-arg pitfall, see - # TestSegmentationConfigWiring.test_flag_reaches_per_variable_trackers), and + # TestClassificationConfigWiring.test_flag_reaches_per_variable_trackers), and # several tests here mutate `detector.config.*` in place -- writing through # to that shared instance and leaking state into any other bare-constructed # CharsetDetector for the rest of the process. Passing a fresh config keeps @@ -390,24 +416,24 @@ def test_returns_none_when_not_configured(self): def test_parses_iso_timestamp(self): detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" - detector.config.timestamp_variable = "ts" + detector.config.auto_config_params.classification = ClassificationMethods(index=False, time=True) + detector.config.auto_config_params.timestamp_variable = "ts" assert detector._timestamp(_parser_record("2026-08-04 10:00:00")) == 1785837600.0 def test_parses_explicit_format(self): """HDFS loghub style, absent from COMMON_TIME_FORMATS.""" detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" - detector.config.timestamp_variable = "ts" - detector.config.timestamp_format = "%y%m%d %H%M%S" + detector.config.auto_config_params.classification = ClassificationMethods(index=False, time=True) + detector.config.auto_config_params.timestamp_variable = "ts" + detector.config.auto_config_params.timestamp_format = "%y%m%d %H%M%S" first = detector._timestamp(_parser_record("081109 203615")) second = detector._timestamp(_parser_record("081109 203645")) assert second - first == 30.0 def test_unparseable_warns_once_and_falls_back(self, caplog): detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" - detector.config.timestamp_variable = "ts" + detector.config.auto_config_params.classification = ClassificationMethods(index=False, time=True) + detector.config.auto_config_params.timestamp_variable = "ts" with caplog.at_level(logging.WARNING): assert detector._timestamp(_parser_record("not-a-time")) is None assert detector._timestamp(_parser_record("also-not-a-time")) is None @@ -415,11 +441,13 @@ def test_unparseable_warns_once_and_falls_back(self, caplog): assert len(warnings) == 1 def test_unset_timestamp_variable_warns_once_and_falls_back(self, caplog): - """stability_segmentation="time" without timestamp_variable is an + """A time-axis classification method without timestamp_variable is an operator error, not an opt-out: it must be distinguishable from a working time-dependent run, and must not flood the log.""" detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" # timestamp_variable left unset + detector.config.auto_config_params.classification = ClassificationMethods( + index=False, time=True + ) # timestamp_variable left unset with caplog.at_level(logging.WARNING): assert detector._timestamp(_parser_record("2026-08-04 10:00:00")) is None assert detector._timestamp(_parser_record("2026-08-04 10:00:01")) is None @@ -436,86 +464,89 @@ def test_flag_off_stays_silent(self, caplog): def test_missing_variable_warns_and_falls_back(self, caplog): detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" - detector.config.timestamp_variable = "absent" + detector.config.auto_config_params.classification = ClassificationMethods(index=False, time=True) + detector.config.auto_config_params.timestamp_variable = "absent" with caplog.at_level(logging.WARNING): assert detector._timestamp(_parser_record("2026-08-04 10:00:00")) is None assert any("timestamp_variable" in r.message for r in caplog.records) -class TestSegmentationConfigWiring: +class TestClassificationConfigWiring: def test_flag_reaches_per_variable_trackers(self): # CharsetDetector's `config` parameter default is a single shared # CharsetDetectorConfig() instance (pre-existing mutable-default-arg - # pitfall, unrelated to stability_segmentation). Other tests in this + # pitfall, unrelated to classification). Other tests in this # module mutate `detector.config.*` in place on a bare # CharsetDetector(), so we pass explicit fresh configs here to stay # isolated from that. + # classification only shapes the configure-phase persistency: the + # trained persistency is read by _check_variable, which never calls + # classify(), so it never receives classification kwargs at all. detector = CharsetDetector(config=CharsetDetectorConfig()) - assert detector.persistency.event_data_kwargs.get("segmentation") is None + assert detector.persistency.event_data_kwargs.get("classification") is None configured = CharsetDetector(config=CharsetDetectorConfig()) - configured.config.stability_segmentation = "time" + configured.config.auto_config_params.classification = ClassificationMethods( + index=False, time=True + ) rebuilt = CharsetDetector(config=configured.config.to_dict(method_id="CharsetDetector")) - assert rebuilt.persistency.event_data_kwargs["segmentation"] == "time" + block = rebuilt.auto_conf_persistency.event_data_kwargs["classification"] + assert ClassificationMethods(**block).enabled == ("time",) def test_config_fields_round_trip(self): detector = CharsetDetector(config=CharsetDetectorConfig()) - detector.config.stability_segmentation = "time" - detector.config.timestamp_variable = "ts" - detector.config.timestamp_format = "%y%m%d %H%M%S" + detector.config.auto_config_params.classification = ClassificationMethods( + index=False, time=True + ) + detector.config.auto_config_params.timestamp_variable = "ts" + detector.config.auto_config_params.timestamp_format = "%y%m%d %H%M%S" restored = type(detector.config).from_dict( detector.config.to_dict(method_id="CharsetDetector"), "CharsetDetector" ) - assert restored.stability_segmentation == "time" - assert restored.timestamp_variable == "ts" - assert restored.timestamp_format == "%y%m%d %H%M%S" - - def test_train_populates_timestamps_end_to_end(self): - cfg = { - "detectors": { - "CharsetDetector": { - "method_type": "charset_detector", - "auto_config": False, - "params": { - "stability_segmentation": "time", - "timestamp_variable": "ts", - "timestamp_format": "%y%m%d %H%M%S", - }, - "events": { - 1: { - "inst": { - "params": {}, - "variables": [{"pos": 0, "name": "v", "params": {}}], - } - } - }, - } - } - } + assert restored.auto_config_params.classification.enabled == ("time",) + assert restored.auto_config_params.timestamp_variable == "ts" + assert restored.auto_config_params.timestamp_format == "%y%m%d %H%M%S" + + def test_configure_populates_timestamps_end_to_end(self): + """The classification block reaches the configure-phase persistency's + trackers end-to-end. + + This used to run through train()/.persistency, but the trained + path no longer receives stability kwargs at all (see + test_train_path_records_no_timestamps) -- configure()/ + .auto_conf_persistency is the phase these settings are for. + """ + cfg = CharsetDetectorConfig( + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods(index=False, time=True), + timestamp_variable="ts", + timestamp_format="%y%m%d %H%M%S", + ), + ) detector = CharsetDetector(config=cfg, name="CharsetDetector") - detector.train(_parser_record("081109 203615")) - detector.train(_parser_record("081109 203645")) - tracker = detector.persistency.get_events_data()[1].get_data()["v"] - assert tracker.segmentation == "time" + detector.configure(_parser_record("081109 203615")) + detector.configure(_parser_record("081109 203645")) + tracker = detector.auto_conf_persistency.get_events_data()[1].get_data()["var_0"] + assert tracker.classification.enabled == ("time",) assert len(tracker.timestamps) == len(tracker.change_series) == 2 assert tracker.timestamps[1] - tracker.timestamps[0] == 30.0 - def test_segmentation_fields_survive_auto_config_set_configuration(self): - """set_configuration() reassigns self.config wholesale from a config - dict generated with empty params (generate_detector_config only emits - method_type/auto_config/params/events), so stability_segmentation, - timestamp_variable and timestamp_format must be carried across that - reassignment explicitly -- same as `persist` already is. + def test_classification_fields_survive_auto_config_set_configuration(self): + """set_configuration() writes only self.config.events and then flips + auto_config to False -- it never rebuilds or reassigns self.config + wholesale, so auto_config_params (like every other operator-set field, + e.g. `persist`) is left untouched by construction. auto_config defaults to True and core.py runs set_configuration() before train(), so this is the path every detector takes unless auto_config is explicitly disabled. """ cfg = CharsetDetectorConfig( - stability_segmentation="time", - timestamp_variable="ts", - timestamp_format="%y%m%d %H%M%S", + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods(index=False, time=True), + timestamp_variable="ts", + timestamp_format="%y%m%d %H%M%S", + ), ) detector = CharsetDetector(config=cfg, name="CharsetDetector") assert detector.config.auto_config is True @@ -524,24 +555,24 @@ def test_segmentation_fields_survive_auto_config_set_configuration(self): detector.configure(_parser_record("081109 203615")) detector.set_configuration() - assert detector.config.stability_segmentation == "time" - assert detector.config.timestamp_variable == "ts" - assert detector.config.timestamp_format == "%y%m%d %H%M%S" + assert detector.config.auto_config_params.classification.enabled == ("time",) + assert detector.config.auto_config_params.timestamp_variable == "ts" + assert detector.config.auto_config_params.timestamp_format == "%y%m%d %H%M%S" -class TestBothSegmentation: - """`both` is STABLE only when count and time segmentation agree.""" +class TestIndexAndTimeTogether: + """Index + time is STABLE only when both methods agree.""" def test_rejects_when_only_time_is_unstable(self): count_mode = SingleStabilityTracker() feed_divergent(count_mode) assert count_mode.classify().type == "STABLE" - time_mode = SingleStabilityTracker(segmentation="time") + time_mode = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) feed_divergent(time_mode) assert time_mode.classify().type == "UNSTABLE" - both_mode = SingleStabilityTracker(segmentation="both") + both_mode = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) feed_divergent(both_mode) assert both_mode.classify().type == "UNSTABLE" @@ -551,18 +582,23 @@ def test_rejects_when_only_count_is_unstable(self): feed_churn(count_mode) assert count_mode.classify().type == "UNSTABLE" - time_mode = SingleStabilityTracker(segmentation="time") + time_mode = SingleStabilityTracker(classification=ClassificationMethods(index=False, time=True)) feed_churn(time_mode) assert time_mode.classify().type == "STABLE" - both_mode = SingleStabilityTracker(segmentation="both") + both_mode = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) feed_churn(both_mode) assert both_mode.classify().type == "UNSTABLE" def test_accepts_when_both_agree(self): """`both` must not be vacuously strict.""" - for mode in ("count", "time", "both"): - tracker = SingleStabilityTracker(segmentation=mode) + modes = { + "index": ClassificationMethods(index=True), + "time": ClassificationMethods(index=False, time=True), + "both": ClassificationMethods(index=True, time=True), + } + for mode, methods in modes.items(): + tracker = SingleStabilityTracker(classification=methods) feed_agreeing(tracker) assert tracker.classify().type == "STABLE", mode @@ -575,7 +611,7 @@ def test_without_timestamps_matches_count_mode(self): also call an all-STABLE fixture STABLE here; only a fixture that is UNSTABLE when fed without timestamps can tell the two apart. """ - both_mode = SingleStabilityTracker(segmentation="both") + both_mode = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) reference = SingleStabilityTracker() for value in CHURN_VALUES: both_mode.add_value(value) # no timestamp argument @@ -595,40 +631,113 @@ def test_reason_reports_both_mean_vectors(self): also yields UNSTABLE, so this exercises the note on the branch finding 1 wires it into. """ - tracker = SingleStabilityTracker(segmentation="both") + tracker = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) feed_churn(tracker) classification = tracker.classify() assert classification.type == "UNSTABLE" reason = classification.reason - assert "count [1.0, 1.0, 1.0, 0.0]" in reason - assert "time [0.9375, 0.0, 0.0, 0.0]" in reason + assert "index: means [1.0, 1.0, 1.0, 0.0]" in reason + assert "time: means [0.9375, 0.0, 0.0, 0.0]" in reason def test_round_trip_preserves_both_mode(self): - tracker = SingleStabilityTracker(segmentation="both") + tracker = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) feed_churn(tracker) restored = SingleStabilityTracker.from_state(tracker.to_state()) - assert restored.segmentation == "both" + assert restored.classification == ClassificationMethods(index=True, time=True) assert restored.timestamps == tracker.timestamps assert restored.classify().type == "UNSTABLE" def test_stability_note_is_not_persisted(self): - tracker = SingleStabilityTracker(segmentation="both") + tracker = SingleStabilityTracker(classification=ClassificationMethods(index=True, time=True)) feed_agreeing(tracker) tracker.classify() assert "_stability_note" not in tracker.to_state() def test_config_accepts_both_and_reaches_trackers(self): + # classification only shapes the configure-phase persistency (see + # TestClassificationConfigWiring.test_flag_reaches_per_variable_trackers). configured = CharsetDetector(config=CharsetDetectorConfig()) - configured.config.stability_segmentation = "both" + configured.config.auto_config_params.classification = ClassificationMethods( + index=True, time=True + ) rebuilt = CharsetDetector( config=configured.config.to_dict(method_id="CharsetDetector") ) - assert rebuilt.persistency.event_data_kwargs["segmentation"] == "both" + block = rebuilt.auto_conf_persistency.event_data_kwargs["classification"] + assert ClassificationMethods(**block).enabled == ("index", "time") def test_event_tracker_propagates_both(self): - event_tracker = EventStabilityTracker(segmentation="both") + event_tracker = EventStabilityTracker( + classification=ClassificationMethods(index=True, time=True) + ) event_tracker.add_data({"var1": "a"}, timestamp=1.0) event_tracker.add_data({"var1": "b"}, timestamp=2.0) single = event_tracker.get_data()["var1"] - assert single.segmentation == "both" + assert single.classification == ClassificationMethods(index=True, time=True) assert single.timestamps == [1.0, 2.0] + + +def test_train_path_records_no_timestamps(): + """Auto-config settings shape the configure-phase persistency only. + + Stability classification is never consulted at detect time, so the + trained trackers would carry an unread timestamps list per variable. + """ + from detectmatelibrary.common.variable_detector import VariableAutoConfigParams + from detectmatelibrary.detectors.new_value_detector import ( + NewValueDetector, + NewValueDetectorConfig, + ) + + detector = NewValueDetector( + name="NewValueDetector", + config=NewValueDetectorConfig( + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods(index=False, time=True), + timestamp_variable="ts", + ), + ), + ) + records = [_parser_record(f"2026-08-04 10:{i:02d}:00") for i in range(20)] + for record in records: + detector.configure(record) + detector.set_configuration() + for record in records: + detector.train(record) + + trained = detector.persistency.get_events_data()[1].get_data() + assert trained, "expected the configure phase to select at least one variable" + for tracker in trained.values(): + assert tracker.classification.enabled == ("index",) + assert tracker.timestamps == [] + + # the configure-phase persistency still gets them + configured = detector.auto_conf_persistency.get_events_data()[1].get_data() + assert any(t.classification.enabled == ("time",) for t in configured.values()) + + +def test_persisted_state_omits_auto_config_params(): + """Persisted tracker state never carries auto_config_params: they are + configure-phase-only inputs, and CharsetDetector's add_value closure + (recovered from `detector_config` on reconstruction, see + _strip_auto_config_params in variable_detector.py) reads only + operational fields, never auto_config_params. + """ + cfg = CharsetDetectorConfig( + auto_config_params=VariableAutoConfigParams( + classification=ClassificationMethods(index=False, time=True), + timestamp_variable="ts", + ), + ) + detector = CharsetDetector(config=cfg, name="CharsetDetector") + for _ in range(5): + detector.configure(_parser_record("2026-08-04 10:00:00")) + detector.set_configuration() + detector.train(_parser_record("2026-08-04 10:00:00")) + + trained = detector.persistency.get_events_data()[1].get_data() + assert trained, "expected the configure phase to select at least one variable" + for tracker in trained.values(): + state = tracker.to_state() + entry = state["detector_config"]["detectors"]["CharsetDetector"] + assert "auto_config_params" not in entry diff --git a/uv.lock b/uv.lock index b029438b..21e65a45 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -272,7 +272,7 @@ requires-dist = [ { name = "detectmatelibrary", extras = ["dataframes"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["llm"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["polars-rtcompat"], marker = "extra == 'full'" }, - { name = "detectmateperformance", specifier = ">=0.1.0" }, + { name = "detectmateperformance", specifier = ">=0.1.5" }, { name = "flax", specifier = ">=0.12.8" }, { name = "fsspec", specifier = ">=2024.1.0" }, { name = "jax", specifier = ">=0.11.0" }, @@ -308,7 +308,7 @@ dev = [ [[package]] name = "detectmateperformance" -version = "0.1.0" +version = "0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "levenshtein" }, @@ -318,9 +318,9 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e8/2b/7ff298303c5ba3ba4899d1f3854b214f63a4fac4eff191012f5608f5149e/detectmateperformance-0.1.0.tar.gz", hash = "sha256:2630d509e7e2bbe6b5bcc857e8f25dfae7d99e9e2cc52f5981da9171600531dc", size = 552409, upload-time = "2026-06-12T10:42:13.326Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/59/92a2666a0062173607e83cf3b1fb9657cbb098c88284102de91202676c6c/detectmateperformance-0.1.5.tar.gz", hash = "sha256:67aa98302a7fc797070a8b0c6eacd147649f1f3df74e24e2753fdf1c0bdaaa1c", size = 826361, upload-time = "2026-08-26T07:28:41.683Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/dd/3db227cd60d170203ab82dd2e670bcb50c6111769c39ac8dd64935fa4c1d/detectmateperformance-0.1.0-py3-none-any.whl", hash = "sha256:db70f239e53aa17a983f99948c9239c143744c03044eeae454f546fedbaf69a5", size = 557939, upload-time = "2026-06-12T10:42:11.526Z" }, + { url = "https://files.pythonhosted.org/packages/65/c3/5386d48d8979655662de51ba1fdf685f6cd92e021efd13bed7a980dc981c/detectmateperformance-0.1.5-py3-none-any.whl", hash = "sha256:82b1b0d8c0163dab66b251a5766c34140a22b683a567b5075b25386bc6cd5eaa", size = 836303, upload-time = "2026-08-26T07:28:39.719Z" }, ] [[package]]