Auto Parser

docs/examples/parsers/auto_parser.py

@@ -0,0 +1,54 @@
+# flake8: noqa
+
+# --8<-- [start:example_1]
+
+from detectmatelibrary.parsers.autoparser import AutoParser
+
+from detectmatelibrary.helper.from_to import From
+
+config_dict = {
+        "parsers": {
+            "AutoParser": {
+                "method_type": "auto_parser",
+                "data_use_training": 10,
+            }
+        }
+    }
+parser = AutoParser(config=config_dict)
+path = "tests/test_data/audit.log"
+
+for j, parsed_log in enumerate(From.log(parser, path)):
+    if j == 15:
+        break
+
+print(parsed_log["template"])  # Returns the parsed log as audit
+
+# --8<-- [end:example_1]
+
+# --8<-- [start:example_2]
+
+from detectmatelibrary.parsers.autoparser import AutoParser
+
+from detectmatelibrary.helper.from_to import From
+
+config_dict = {
+        "parsers": {
+            "AutoParser": {
+                "method_type": "auto_parser",
+                "data_use_training": 10,
+                "params": {
+                    "fix_type": "Audit"  # Fix search to only Audit
+                }
+            }
+        }
+    }
+parser = AutoParser(config=config_dict)
+path = "tests/test_data/audit.log"
+
+for j, parsed_log in enumerate(From.log(parser, path)):
+    if j == 15:
+        break
+
+print(parsed_log["template"])  # Returns the parsed log as audit
+
+# --8<-- [end:example_2]

docs/parsers.md

@@ -108,5 +108,6 @@ def test_my_parser_parse():
 - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates.
 - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure.
 - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data.
+- [Auto Parser](parsers/auto_parser.md): extract the logs from the templates saved in the dataset.
 
 Go back to [Index](index.md)

docs/parsers/auto_parser.md

@@ -0,0 +1,34 @@
+# Auto Parser
+
+Parse the logs using the templates saved in the dataset. (HDFS, BGL, Audit, SysLog, Apache, OpenVPN, Thunderbird).
+
+It wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. When parsing large numbers of log lines in non-stream (batch) mode, it is recommended to use the performance-oriented implementation.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [LogSchema](../schemas.md) | Unstructured log   |
+| **Output** | [ParserSchema](../schemas.md) | Structured log   |
+
+## Configuration
+
+Auto parser parameters:
+
+- `method_type` (string): identifier for the parser type (e.g., `"aut_parser"`).
+- `fix_type` (str): fix type of logs to process.
+
+
+## Usage example
+
+Without fixing log type:
+
+```python
+--8<-- "docs/examples/parsers/auto_parser.py:example_1"
+```
+
+With fixing log type:
+
+```python
+--8<-- "docs/examples/parsers/auto_parser.py:example_2"
+```
+
+Go back to [Index](../index.md)

mkdocs.yml

@@ -23,6 +23,7 @@ nav:
     - Template Tree Matcher: parsers/template_tree_matcher.md
     - Json Parser: parsers/json_parser.md
     - LogBatcher Parser: parsers/logbatcher_parser.md
+    - Auto Parser: parsers/auto_parser.md
   - Detectors Methods:
     - Random Detector: detectors/random_detector.md
     - New Value: detectors/new_value.md

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
     "numpy>=2.3.2",
-    "detectmateperformance>=0.1.0",
+    "detectmateperformance>=0.1.6",
     "msgpack>=1.0.0",
     "fsspec>=2024.1.0",
     "pyarrow>=24.0.0",

src/detectmatelibrary/parsers/autoparser.py

@@ -0,0 +1,59 @@
+from detectmatelibrary.common.parser import CoreParser, CoreParserConfig
+from detectmatelibrary import schemas
+
+from detectmateperformance.match_tree import TreeMatcher
+from detectmateperformance.autoparser import AutoParse
+
+
+from typing import Any
+import warnings
+import re
+
+
+class AutoParserConfig(CoreParserConfig):
+    method_type: str = "auto_parser"
+    fix_type: str = ""
+
+
+class AutoParser(CoreParser):
+    def __init__(
+        self,
+        name: str = "AutoParser",
+        config: AutoParserConfig | dict[str, Any] = AutoParserConfig()
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = AutoParserConfig.from_dict(config, name)
+        super().__init__(name=name, config=config)
+
+        self.config: AutoParserConfig
+        self.auto_gen = AutoParse(num_use=self.config.data_use_training)
+        self.tree_match: TreeMatcher | None = None
+
+        self.config_buffer: list[str] = []
+
+    def train(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.auto_gen.add(input_["log"])
+
+    def post_train(self) -> None:
+        try:
+            self.tree_match, _regex = self.auto_gen.generate(self.config.fix_type)
+            self.config._regex = re.compile(_regex)
+        except RuntimeError as e:
+            warnings.warn(str(e))
+
+    def parse(
+        self,
+        input_: schemas.LogSchema,
+        output_: schemas.ParserSchema
+    ) -> None:
+
+        if self.tree_match is None:
+            output_["EventID"] = -1
+            output_["template"] = "templates not yet generated"
+        else:
+            parsed = self.tree_match.match_log(input_["log"], get_var=True)[0]
+
+            output_["EventID"] = parsed["EventID"]
+            output_["variables"].extend(parsed["ParamList"])
+            output_["template"] = parsed["Template"]

tests/test_parsers/test_autoparser.py

@@ -0,0 +1,67 @@
+"""Most of the functionality is test it in DetectMatePerformance."""
+from detectmatelibrary.parsers.autoparser import AutoParser
+
+from detectmatelibrary.helper.from_to import From
+import pytest
+
+temp = "pid <*> uid <*> auid <*> ses <*> msg op <*> acct <*> exe <*> hostname <*> addr <*> terminal <*> res <*>"  # noqa: E501
+
+
+class TestAutoParserParser:
+    def test_train_process(self):
+        config_dict = {
+            "parsers": {
+                "AutoParser": {
+                    "method_type": "auto_parser",
+                    "data_use_training": 10,
+                }
+            }
+        }
+        parser = AutoParser(config=config_dict)
+        path = "tests/test_data/audit.log"
+
+        for j, parsed_log in enumerate(From.log(parser, path)):
+            if j == 15:
+                break
+
+        assert parsed_log["template"] == temp
+
+    def test_fix_templates(self):
+        config_dict = {
+            "parsers": {
+                "AutoParser": {
+                    "method_type": "auto_parser",
+                    "data_use_training": 10,
+                    "params": {
+                        "fix_type": "BGL"
+                    }
+                }
+            }
+        }
+        parser = AutoParser(config=config_dict)
+        path = "tests/test_data/audit.log"
+
+        for j, parsed_log in enumerate(From.log(parser, path)):
+            if j == 15:
+                break
+
+        assert parsed_log["template"] == "template not found"
+
+    def test_unknonw_fix_dataset(self):
+        config_dict = {
+            "parsers": {
+                "AutoParser": {
+                    "method_type": "auto_parser",
+                    "data_use_training": 1,
+                    "params": {
+                        "fix_type": "Unknown"
+                    }
+                }
+            }
+        }
+        parser = AutoParser(config=config_dict)
+        path = "tests/test_data/audit.log"
+
+        next(From.log(parser, path))
+        with pytest.warns():
+            next(From.log(parser, path))