diff --git a/docs/examples/parsers/drain_parser.py b/docs/examples/parsers/drain_parser.py new file mode 100644 index 00000000..cfd949a4 --- /dev/null +++ b/docs/examples/parsers/drain_parser.py @@ -0,0 +1,70 @@ +# flake8: noqa + +# --8<-- [start:example_1] +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" +# --8<-- [end:example_1] + + +# --8<-- [start:example_2] +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "template not found" +# --8<-- [end:example_2] diff --git a/docs/parsers.md b/docs/parsers.md index a862c877..2db5fc08 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -108,5 +108,6 @@ def test_my_parser_parse(): - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates. - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure. - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data. +- [Drain parser](parsers/drain_parser.md): Parser inspired by [Drain Publication](https://ieeexplore.ieee.org/document/8029742). Go back to [Index](index.md) diff --git a/docs/parsers/drain_parser.md b/docs/parsers/drain_parser.md new file mode 100644 index 00000000..f20c6afb --- /dev/null +++ b/docs/parsers/drain_parser.md @@ -0,0 +1,47 @@ +# Drain parser + +The parser is derived from the official [Drain publication](https://ieeexplore.ieee.org/document/8029742). + +It also wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. When parsing large numbers of log lines in non-stream (batch) mode, it is recommended to use the performance-oriented implementation. + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [LogSchema](../schemas.md) | Unstructured log | +| **Output** | [ParserSchema](../schemas.md) | Structured log | + +## Configuration + +Drain parser parameters: + +- `method_type` (string): identifier for the parser type (e.g., `"tree_matcher"`). +- `depth` (int): number of token/word levels. +- `max_childs` (int): maximum number of children allowed in the given layer. +- `sim_thres` (float): threshold used for similarity. +- `reset_in_post_train` (bool): if enabled, clears logs from the training buffer once templates are created; otherwise, it keeps them for the next training cycle. +- `auto_config` (bool): indicates whether to run an optional auto-configuration step (not mandatory). + +Example YAML fragment: +```yaml +parsers: + DrainParser: + method_type: drain_parser + auto_config: False + params: + depth: 2 +``` + +## Usage example + +Simple usage (Reset = False): + +```python +--8<-- "docs/examples/parsers/drain_parser.py:example_1" +``` + +Simple usage (Reset = True): + +```python +--8<-- "docs/examples/parsers/drain_parser.py:example_2" +``` + +Go back to [Index](../index.md) diff --git a/mkdocs.yml b/mkdocs.yml index d44b3fbb..60e4e8fc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Template Tree Matcher: parsers/template_tree_matcher.md - Json Parser: parsers/json_parser.md - LogBatcher Parser: parsers/logbatcher_parser.md + - Drain Parser: parsers/drain_parser.md - Detectors Methods: - Random Detector: detectors/random_detector.md - New Value: detectors/new_value.md diff --git a/pyproject.toml b/pyproject.toml index 857ae87b..b29abd9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pyyaml>=6.0.3", "regex>=2025.11.3", "numpy>=2.3.2", - "detectmateperformance>=0.1.0", + "detectmateperformance>=0.1.5", "msgpack>=1.0.0", "fsspec>=2024.1.0", "pyarrow>=24.0.0", diff --git a/src/detectmatelibrary/parsers/drain.py b/src/detectmatelibrary/parsers/drain.py index e69de29b..abed9424 100644 --- a/src/detectmatelibrary/parsers/drain.py +++ b/src/detectmatelibrary/parsers/drain.py @@ -0,0 +1,113 @@ +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig +from detectmatelibrary import schemas + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.drain import Drain + +from detectmatelibrary.utils.finetune import Combinations + +from typing import Any + + +class DrainConfig(CoreParserConfig): + method_type: str = "drain_parser" + + depth: int = 2 + max_childs: int = 10 + sim_thres: float = 0.2 + + reset_in_post_train: bool = False + + Finetune: list[list[str | list[Any]]] = [ + ["depth", [1, 2, 3, 4]], + ["max_childs", [10, 40]], + ["sim_thres", [0.2, 0.4, 0.6, 0.8]] + ] + + +def _init_drain(config: DrainConfig) -> Drain: + return Drain( + depth=config.depth, max_child=config.max_childs, sim=config.sim_thres, + ) + + +def _found_ratio(logs: list[str], tree_matcher: TreeMatcher) -> float: + results = tree_matcher.match_batch(logs).get_all_templates() + + score = 0.0 + for template in results: + if "template not found" == template: + score += 1. + + return score / len(results) + + +def _get_best_config(logs: list[str], config: DrainConfig) -> DrainConfig: + + found_ratio: list[float] = [] + length: list[int] = [] + + for config in (comb := Combinations(config))(): # type: ignore + drain = _init_drain(config) + for input_ in logs: + drain.add(input_) + tree_matcher = drain.generate() + + found_ratio.append(_found_ratio(logs, tree_matcher)) + length.append(len(tree_matcher)) + + n = max(length) + for le, sc in zip(length, found_ratio): + comb.add_value((float(le) / n) + sc) + + new_config: DrainConfig = comb.get_best() # type: ignore + return new_config + + +class DrainParser(CoreParser): + def __init__( + self, + name: str = "DrainParser", + config: DrainConfig | dict[str, Any] = DrainConfig() + ) -> None: + + if isinstance(config, dict): + config = DrainConfig.from_dict(config, name) + super().__init__(name=name, config=config) + + self.config: DrainConfig + self.drain_gen = _init_drain(config=config) + self.tree_match: TreeMatcher | None = None + + self.config_buffer: list[str] = [] + + def configure(self, input_: schemas.LogSchema) -> None: # type: ignore + self.config_buffer.append(input_["log"]) + + def set_configuration(self) -> None: + self.config = _get_best_config(self.config_buffer, config=self.config) + self.config_buffer = [] + + def train(self, input_: schemas.LogSchema) -> None: # type: ignore + self.drain_gen.add(input_["log"]) + + def post_train(self) -> None: + self.tree_match = self.drain_gen.generate() + if self.config.reset_in_post_train: + self.drain_gen.reset() + + def parse( + self, + input_: schemas.LogSchema, + output_: schemas.ParserSchema + ) -> None: + + if self.tree_match is None: + output_["EventID"] = -1 + output_["template"] = "templates not yet generated" + else: + parsed = self.tree_match.match_log(input_["log"], get_var=True)[0] + + output_["EventID"] = parsed["EventID"] + output_["variables"].extend(parsed["ParamList"]) + output_["template"] = parsed["Template"] diff --git a/tests/test_parsers/test_drain.py b/tests/test_parsers/test_drain.py new file mode 100644 index 00000000..07aebb9b --- /dev/null +++ b/tests/test_parsers/test_drain.py @@ -0,0 +1,140 @@ +"""Most of the functionality is test it in DetectMatePerformance.""" +from detectmatelibrary.parsers.drain import DrainParser, _found_ratio + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.types_ import LogTemplates + +from detectmatelibrary import schemas + + +class TestDrainParser: + def test_train_process(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general R2D2!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_reset_after_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": True, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_not_reset_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": False, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["template"] == "hello there <*> kenobi" + + def test_not_ration_found(self): + tree_matcher = TreeMatcher(LogTemplates(["hello there <*> kenobi"])) + + logs = ["hello there general kenobi", "akuna matata"] + assert 0.5 == _found_ratio(logs, tree_matcher) + + logs = ["hello there general kenobi"] + assert 0. == _found_ratio(logs, tree_matcher) + + def test_no_auto_config_but_no_initialization(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "auto_config": True, + "data_use_configure": 2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + + assert parser.config.depth == 1 + assert parser.config.max_childs == 10 + assert parser.config.sim_thres == 0.2 diff --git a/uv.lock b/uv.lock index b029438b..21e65a45 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -272,7 +272,7 @@ requires-dist = [ { name = "detectmatelibrary", extras = ["dataframes"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["llm"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["polars-rtcompat"], marker = "extra == 'full'" }, - { name = "detectmateperformance", specifier = ">=0.1.0" }, + { name = "detectmateperformance", specifier = ">=0.1.5" }, { name = "flax", specifier = ">=0.12.8" }, { name = "fsspec", specifier = ">=2024.1.0" }, { name = "jax", specifier = ">=0.11.0" }, @@ -308,7 +308,7 @@ dev = [ [[package]] name = "detectmateperformance" -version = "0.1.0" +version = "0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "levenshtein" }, @@ -318,9 +318,9 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e8/2b/7ff298303c5ba3ba4899d1f3854b214f63a4fac4eff191012f5608f5149e/detectmateperformance-0.1.0.tar.gz", hash = "sha256:2630d509e7e2bbe6b5bcc857e8f25dfae7d99e9e2cc52f5981da9171600531dc", size = 552409, upload-time = "2026-06-12T10:42:13.326Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/59/92a2666a0062173607e83cf3b1fb9657cbb098c88284102de91202676c6c/detectmateperformance-0.1.5.tar.gz", hash = "sha256:67aa98302a7fc797070a8b0c6eacd147649f1f3df74e24e2753fdf1c0bdaaa1c", size = 826361, upload-time = "2026-08-26T07:28:41.683Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/dd/3db227cd60d170203ab82dd2e670bcb50c6111769c39ac8dd64935fa4c1d/detectmateperformance-0.1.0-py3-none-any.whl", hash = "sha256:db70f239e53aa17a983f99948c9239c143744c03044eeae454f546fedbaf69a5", size = 557939, upload-time = "2026-06-12T10:42:11.526Z" }, + { url = "https://files.pythonhosted.org/packages/65/c3/5386d48d8979655662de51ba1fdf685f6cd92e021efd13bed7a980dc981c/detectmateperformance-0.1.5-py3-none-any.whl", hash = "sha256:82b1b0d8c0163dab66b251a5766c34140a22b683a567b5075b25386bc6cd5eaa", size = 836303, upload-time = "2026-08-26T07:28:39.719Z" }, ] [[package]]