Feat/parser dp

docs/examples/parsers/drain_parser.py

@@ -0,0 +1,70 @@
+# flake8: noqa
+
+# --8<-- [start:example_1]
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+# --8<-- [end:example_1]
+
+
+# --8<-- [start:example_2]
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "template not found"
+# --8<-- [end:example_2]

docs/parsers.md

@@ -108,5 +108,6 @@ def test_my_parser_parse():
 - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates.
 - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure.
 - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data.
+- [Drain parser](parsers/drain_parser.md): Parser inspired by [Drain Publication](https://ieeexplore.ieee.org/document/8029742).
 
 Go back to [Index](index.md)

docs/parsers/drain_parser.md

@@ -0,0 +1,47 @@
+# Drain parser
+
+The parser is derived from the official [Drain publication](https://ieeexplore.ieee.org/document/8029742).
+
+It also wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. When parsing large numbers of log lines in non-stream (batch) mode, it is recommended to use the performance-oriented implementation.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [LogSchema](../schemas.md) | Unstructured log   |
+| **Output** | [ParserSchema](../schemas.md) | Structured log   |
+
+## Configuration
+
+Drain parser parameters:
+
+- `method_type` (string): identifier for the parser type (e.g., `"tree_matcher"`).
+- `depth` (int): number of token/word levels.
+- `max_childs` (int): maximum number of children allowed in the given layer.
+- `sim_thres` (float): threshold used for similarity.
+- `reset_in_post_train` (bool): if enabled, clears logs from the training buffer once templates are created; otherwise, it keeps them for the next training cycle.
+- `auto_config` (bool): indicates whether to run an optional auto-configuration step (not mandatory).
+
+Example YAML fragment:
+```yaml
+parsers:
+  DrainParser:
+    method_type: drain_parser
+    auto_config: False
+    params:
+      depth: 2
+```
+
+## Usage example
+
+Simple usage (Reset = False):
+
+```python
+--8<-- "docs/examples/parsers/drain_parser.py:example_1"
+```
+
+Simple usage (Reset = True):
+
+```python
+--8<-- "docs/examples/parsers/drain_parser.py:example_2"
+```
+
+Go back to [Index](../index.md)

mkdocs.yml

@@ -22,6 +22,7 @@ nav:
     - Template Tree Matcher: parsers/template_tree_matcher.md
     - Json Parser: parsers/json_parser.md
     - LogBatcher Parser: parsers/logbatcher_parser.md
+    - Drain Parser: parsers/drain_parser.md
   - Detectors Methods:
     - Random Detector: detectors/random_detector.md
     - New Value: detectors/new_value.md

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
     "numpy>=2.3.2",
-    "detectmateperformance>=0.1.0",
+    "detectmateperformance>=0.1.5",
     "msgpack>=1.0.0",
     "fsspec>=2024.1.0",
     "pyarrow>=24.0.0",

src/detectmatelibrary/parsers/drain.py

@@ -0,0 +1,113 @@
+from detectmatelibrary.common.parser import CoreParser, CoreParserConfig
+from detectmatelibrary import schemas
+
+from detectmateperformance.match_tree import TreeMatcher
+from detectmateperformance.drain import Drain
+
+from detectmatelibrary.utils.finetune import Combinations
+
+from typing import Any
+
+
+class DrainConfig(CoreParserConfig):
+    method_type: str = "drain_parser"
+
+    depth: int = 2
+    max_childs: int = 10
+    sim_thres: float = 0.2
+
+    reset_in_post_train: bool = False
+
+    Finetune: list[list[str | list[Any]]] = [
+        ["depth", [1, 2, 3, 4]],
+        ["max_childs", [10, 40]],
+        ["sim_thres", [0.2, 0.4, 0.6, 0.8]]
+    ]
+
+
+def _init_drain(config: DrainConfig) -> Drain:
+    return Drain(
+        depth=config.depth, max_child=config.max_childs, sim=config.sim_thres,
+    )
+
+
+def _found_ratio(logs: list[str], tree_matcher: TreeMatcher) -> float:
+    results = tree_matcher.match_batch(logs).get_all_templates()
+
+    score = 0.0
+    for template in results:
+        if "template not found" == template:
+            score += 1.
+
+    return score / len(results)
+
+
+def _get_best_config(logs: list[str], config: DrainConfig) -> DrainConfig:
+
+    found_ratio: list[float] = []
+    length: list[int] = []
+
+    for config in (comb := Combinations(config))():  # type: ignore
+        drain = _init_drain(config)
+        for input_ in logs:
+            drain.add(input_)
+        tree_matcher = drain.generate()
+
+        found_ratio.append(_found_ratio(logs, tree_matcher))
+        length.append(len(tree_matcher))
+
+    n = max(length)
+    for le, sc in zip(length, found_ratio):
+        comb.add_value((float(le) / n) + sc)
+
+    new_config: DrainConfig = comb.get_best()  # type: ignore
+    return new_config
+
+
+class DrainParser(CoreParser):
+    def __init__(
+        self,
+        name: str = "DrainParser",
+        config: DrainConfig | dict[str, Any] = DrainConfig()
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = DrainConfig.from_dict(config, name)
+        super().__init__(name=name, config=config)
+
+        self.config: DrainConfig
+        self.drain_gen = _init_drain(config=config)
+        self.tree_match: TreeMatcher | None = None
+
+        self.config_buffer: list[str] = []
+
+    def configure(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.config_buffer.append(input_["log"])
+
+    def set_configuration(self) -> None:
+        self.config = _get_best_config(self.config_buffer, config=self.config)
+        self.config_buffer = []
+
+    def train(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.drain_gen.add(input_["log"])
+
+    def post_train(self) -> None:
+        self.tree_match = self.drain_gen.generate()
+        if self.config.reset_in_post_train:
+            self.drain_gen.reset()
+
+    def parse(
+        self,
+        input_: schemas.LogSchema,
+        output_: schemas.ParserSchema
+    ) -> None:
+
+        if self.tree_match is None:
+            output_["EventID"] = -1
+            output_["template"] = "templates not yet generated"
+        else:
+            parsed = self.tree_match.match_log(input_["log"], get_var=True)[0]
+
+            output_["EventID"] = parsed["EventID"]
+            output_["variables"].extend(parsed["ParamList"])
+            output_["template"] = parsed["Template"]

tests/test_parsers/test_drain.py

@@ -0,0 +1,140 @@
+"""Most of the functionality is test it in DetectMatePerformance."""
+from detectmatelibrary.parsers.drain import DrainParser, _found_ratio
+
+from detectmateperformance.match_tree import TreeMatcher
+from detectmateperformance.types_ import LogTemplates
+
+from detectmatelibrary import schemas
+
+
+class TestDrainParser:
+    def test_train_process(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "depth": 2,
+                    "max_childs": 10,
+                    "sim_thres": 0.2,
+                    "data_use_training": 2,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general R2D2!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "template not found"
+
+    def test_reset_after_train(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "depth": 2,
+                    "max_childs": 10,
+                    "sim_thres": 0.2,
+                    "data_use_training": 2,
+                    "reset_in_post_train": True,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parser.update_state("keep_training")
+        parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+        parser.update_state("stop_training")
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "template not found"
+
+    def test_not_reset_train(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "depth": 2,
+                    "max_childs": 10,
+                    "sim_thres": 0.2,
+                    "data_use_training": 2,
+                    "reset_in_post_train": False,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        assert parsed["EventID"] == -1
+        assert parsed["template"] == "templates not yet generated"
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["EventID"] == 0
+        assert parsed["template"] == "hello there <*> kenobi"
+
+        parser.update_state("keep_training")
+        parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+        parser.update_state("stop_training")
+
+        parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+        assert parsed["template"] == "hello there <*> kenobi"
+
+    def test_not_ration_found(self):
+        tree_matcher = TreeMatcher(LogTemplates(["hello there <*> kenobi"]))
+
+        logs = ["hello there general kenobi", "akuna matata"]
+        assert 0.5 == _found_ratio(logs, tree_matcher)
+
+        logs = ["hello there general kenobi"]
+        assert 0. == _found_ratio(logs, tree_matcher)
+
+    def test_no_auto_config_but_no_initialization(self):
+        config_dict = {
+            "parsers": {
+                "DrainParser": {
+                    "method_type": "drain_parser",
+                    "depth": 2,
+                    "max_childs": 10,
+                    "sim_thres": 0.2,
+                    "auto_config": True,
+                    "data_use_configure": 2,
+                    "data_use_training": 2,
+                }
+            }
+        }
+        parser = DrainParser(config=config_dict)
+        parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+        parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+        parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+
+        assert parser.config.depth == 1
+        assert parser.config.max_childs == 10
+        assert parser.config.sim_thres == 0.2