diff --git a/converters/dbt/src/ossie_dbt/msi_to_ossie.py b/converters/dbt/src/ossie_dbt/msi_to_ossie.py index 3132cdcb..a887ad44 100644 --- a/converters/dbt/src/ossie_dbt/msi_to_ossie.py +++ b/converters/dbt/src/ossie_dbt/msi_to_ossie.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import re from collections import defaultdict from dataclasses import dataclass @@ -22,6 +23,7 @@ from typing import Dict, List, Optional, Sequence, Tuple from ossie import ( + OssieCustomExtension, OssieDataset, OssieDialect, OssieDialectExpression, @@ -32,6 +34,7 @@ OssieMetric, OssieRelationship, OssieSemanticModel, + OssieVendor, ) from ossie_dbt.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult from ossie_dbt.filter_utils import _collect_filter_sql, _merge_filter_sqls @@ -44,6 +47,7 @@ Measure, MeasureAggregationParameters, ) +from metricflow_semantic_interfaces.protocols.meta import SemanticLayerElementConfig from metricflow_semantic_interfaces.protocols.metric import Metric from metricflow_semantic_interfaces.protocols.semantic_model import SemanticModel from metricflow_semantic_interfaces.transformations.convert_count import ConvertCountMetricToSumRule @@ -116,6 +120,7 @@ def convert( name=metric.name, expression=self._make_expression(expr), description=metric.description, + custom_extensions=self._custom_extensions_from_meta(metric.config), ) ) @@ -153,6 +158,7 @@ def _convert_semantic_model(self, sm: SemanticModel) -> OssieDataset: unique_keys=unique_keys if unique_keys else None, description=sm.description, fields=fields if fields else None, + custom_extensions=self._custom_extensions_from_meta(sm.config), ) def _convert_dimension(self, dim: Dimension) -> OssieField: @@ -165,6 +171,7 @@ def _convert_dimension(self, dim: Dimension) -> OssieField: dimension=OssieDimension(is_time=is_time), label=dim.label, description=dim.description, + custom_extensions=self._custom_extensions_from_meta(dim.config), ) def _convert_entity(self, entity: Entity) -> OssieField: @@ -175,6 +182,7 @@ def _convert_entity(self, entity: Entity) -> OssieField: expression=self._make_expression(expr), label=entity.label, description=entity.description, + custom_extensions=self._custom_extensions_from_meta(entity.config), ) def _convert_measure(self, measure: Measure) -> OssieField: @@ -185,8 +193,21 @@ def _convert_measure(self, measure: Measure) -> OssieField: expression=self._make_expression(expr), label=measure.label, description=measure.description, + custom_extensions=self._custom_extensions_from_meta(measure.config), ) + @staticmethod + def _custom_extensions_from_meta( + config: Optional[SemanticLayerElementConfig], + ) -> Optional[List[OssieCustomExtension]]: + """Carry a non-empty MSI `config.meta` dict across as one Ossie `custom_extensions` entry. + + `config.meta` is otherwise silently dropped at this boundary (ossie#303). + """ + if config is None or not config.meta: + return None + return [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps(config.meta))] + @staticmethod def _extract_keys(entities: Sequence[Entity]) -> Tuple[Optional[List[str]], List[List[str]]]: primary_key: Optional[List[str]] = None diff --git a/converters/dbt/src/ossie_dbt/ossie_to_msi.py b/converters/dbt/src/ossie_dbt/ossie_to_msi.py index dfad90b0..5d82f353 100644 --- a/converters/dbt/src/ossie_dbt/ossie_to_msi.py +++ b/converters/dbt/src/ossie_dbt/ossie_to_msi.py @@ -15,16 +15,19 @@ # specific language governing permissions and limitations # under the License. +import json from dataclasses import dataclass from typing import List, Optional, Set from ossie import ( + OssieCustomExtension, OssieDataset, OssieDialect, OssieDocument, OssieExpression, OssieField, OssieSemanticModel, + OssieVendor, ) from ossie_dbt.converter_issues import ConverterResult from ossie_dbt.expression_utils import ( @@ -34,6 +37,9 @@ _try_parse_ratio, ) +from metricflow_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) from metricflow_semantic_interfaces.implementations.elements.dimension import ( PydanticDimension, PydanticDimensionTypeParams, @@ -149,8 +155,23 @@ def _convert_dataset( entities=entities, dimensions=dimensions, measures=[], + config=self._config_from_custom_extensions(dataset.custom_extensions), ) + @staticmethod + def _config_from_custom_extensions( + custom_extensions: Optional[List[OssieCustomExtension]], + ) -> Optional[PydanticSemanticLayerElementConfig]: + """Reconstruct MSI `config.meta` from the `custom_extensions` entry this converter wrote (ossie#303). + + Only the DBT-vendor entry round-trips; extensions written by another vendor did not + originate from config.meta and have no meaning as one. + """ + for ext in custom_extensions or []: + if ext.vendor_name == OssieVendor.DBT.value: + return PydanticSemanticLayerElementConfig(meta=json.loads(ext.data)) + return None + @staticmethod def _build_key_sets(dataset: OssieDataset, ossie_sm: OssieSemanticModel) -> _KeySets: """Return a _KeySets with primary, unique, and foreign key column sets for a dataset.""" @@ -197,7 +218,7 @@ def _classify_field( description=field.description, label=field.label, role=None, - config=None, + config=self._config_from_custom_extensions(field.custom_extensions), ) ) return @@ -210,7 +231,7 @@ def _classify_field( description=field.description, label=field.label, role=None, - config=None, + config=self._config_from_custom_extensions(field.custom_extensions), ) ) return @@ -223,7 +244,7 @@ def _classify_field( description=field.description, label=field.label, role=None, - config=None, + config=self._config_from_custom_extensions(field.custom_extensions), ) ) return @@ -237,7 +258,7 @@ def _classify_field( expr=expr_or_none, description=field.description, label=field.label, - config=None, + config=self._config_from_custom_extensions(field.custom_extensions), ) ) return @@ -249,7 +270,7 @@ def _classify_field( expr=expr_or_none, description=field.description, label=field.label, - config=None, + config=self._config_from_custom_extensions(field.custom_extensions), ) ) @@ -261,7 +282,15 @@ def _convert_metrics(self, ossie_sm: OssieSemanticModel) -> List[PydanticMetric] metrics: List[PydanticMetric] = [] for metric in ossie_sm.metrics or []: expr_str = self._get_expression(metric.expression) - metrics.extend(self._convert_metric(metric.name, expr_str, metric.description, ossie_sm.datasets)) + metrics.extend( + self._convert_metric( + metric.name, + expr_str, + metric.description, + ossie_sm.datasets, + config=self._config_from_custom_extensions(metric.custom_extensions), + ) + ) return metrics def _convert_metric( @@ -270,6 +299,7 @@ def _convert_metric( expr_str: str, description: Optional[str], datasets: List[OssieDataset], + config: Optional[PydanticSemanticLayerElementConfig] = None, ) -> List[PydanticMetric]: """Return one or more PydanticMetric objects for the given Ossie expression. @@ -302,7 +332,7 @@ def _convert_metric( ), filter=None, metadata=None, - config=None, + config=config, ) ] @@ -324,7 +354,7 @@ def _convert_metric( ), filter=None, metadata=None, - config=None, + config=config, ) return [*num_metrics, *den_metrics, ratio_metric] @@ -349,7 +379,7 @@ def _convert_metric( ), filter=None, metadata=None, - config=None, + config=config, ) ] diff --git a/converters/dbt/tests/helpers.py b/converters/dbt/tests/helpers.py index 644d53ba..ffeaf325 100644 --- a/converters/dbt/tests/helpers.py +++ b/converters/dbt/tests/helpers.py @@ -18,6 +18,7 @@ """Shared test helpers for Ossie converter tests.""" from ossie import ( + OssieCustomExtension, OssieDataset, OssieDialect, OssieDialectExpression, @@ -29,6 +30,9 @@ OssieRelationship, OssieSemanticModel, ) +from metricflow_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) from metricflow_semantic_interfaces.implementations.elements.dimension import ( PydanticDimension, PydanticDimensionTypeParams, @@ -79,6 +83,7 @@ def _simple_metric( name: str, measure_name: str, description: str | None = None, + config: PydanticSemanticLayerElementConfig | None = None, ) -> PydanticMetric: return PydanticMetric( name=name, @@ -89,7 +94,7 @@ def _simple_metric( ), filter=None, metadata=default_meta(), - config=None, + config=config, ) @@ -100,6 +105,7 @@ def _dimension( description: str | None = None, label: str | None = None, granularity: TimeGranularity | None = None, + config: PydanticSemanticLayerElementConfig | None = None, ) -> PydanticDimension: type_params = PydanticDimensionTypeParams(time_granularity=granularity) if granularity else None return PydanticDimension( @@ -110,7 +116,7 @@ def _dimension( label=label, type_params=type_params, metadata=default_meta(), - config=None, + config=config, ) @@ -120,6 +126,7 @@ def _measure( expr: str | None = None, description: str | None = None, label: str | None = None, + config: PydanticSemanticLayerElementConfig | None = None, ) -> PydanticMeasure: return PydanticMeasure( name=name, @@ -130,6 +137,7 @@ def _measure( create_metric=None, agg_params=None, metadata=default_meta(), + config=config, ) @@ -137,6 +145,7 @@ def _entity( name: str, entity_type: EntityType = EntityType.PRIMARY, expr: str | None = None, + config: PydanticSemanticLayerElementConfig | None = None, ) -> PydanticEntity: return PydanticEntity( name=name, @@ -144,7 +153,7 @@ def _entity( expr=expr, description=None, role=None, - config=None, + config=config, ) @@ -167,6 +176,7 @@ def _ossie_field( is_time: bool | None = None, description: str | None = None, label: str | None = None, + custom_extensions: list[OssieCustomExtension] | None = None, ) -> OssieField: return OssieField( name=name, @@ -174,6 +184,7 @@ def _ossie_field( dimension=OssieDimension(is_time=is_time) if is_time is not None else None, description=description, label=label, + custom_extensions=custom_extensions, ) @@ -184,6 +195,7 @@ def _ossie_dataset( primary_key: list[str] | None = None, unique_keys: list[list[str]] | None = None, description: str | None = None, + custom_extensions: list[OssieCustomExtension] | None = None, ) -> OssieDataset: return OssieDataset( name=name, @@ -192,11 +204,19 @@ def _ossie_dataset( primary_key=primary_key, unique_keys=unique_keys, description=description, + custom_extensions=custom_extensions, ) -def _ossie_metric(name: str, expression: str, description: str | None = None) -> OssieMetric: - return OssieMetric(name=name, expression=_ossie_expr(expression), description=description) +def _ossie_metric( + name: str, + expression: str, + description: str | None = None, + custom_extensions: list[OssieCustomExtension] | None = None, +) -> OssieMetric: + return OssieMetric( + name=name, expression=_ossie_expr(expression), description=description, custom_extensions=custom_extensions + ) def _ossie_relationship( diff --git a/converters/dbt/tests/test_msi_to_ossie.py b/converters/dbt/tests/test_msi_to_ossie.py index 98102628..bad9eb6c 100644 --- a/converters/dbt/tests/test_msi_to_ossie.py +++ b/converters/dbt/tests/test_msi_to_ossie.py @@ -25,6 +25,9 @@ from ossie_dbt.filter_utils import _render_filter_template from ossie import OssieDialect, OssieDocument from ossie_dbt.msi_to_ossie import MSIToOssieConverter +from metricflow_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) from metricflow_semantic_interfaces.implementations.metric import ( PydanticConversionTypeParams, PydanticCumulativeTypeParams, @@ -249,6 +252,111 @@ def test_foreign_entity_also_becomes_field(self) -> None: assert _fields(result)[0].name == "user_id" +class TestConfigMetaCustomExtensions: + """config.meta is otherwise silently dropped at the MSI -> Ossie boundary (ossie#303).""" + + def test_dimension_config_meta_becomes_custom_extension(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + dimensions=[ + _dimension( + "status", + config=PydanticSemanticLayerElementConfig(meta={"looker_group_label": "Order Info"}), + ) + ], + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + extensions = _fields(result)[0].custom_extensions + assert extensions is not None + assert len(extensions) == 1 + assert extensions[0].vendor_name == "DBT" + assert json.loads(extensions[0].data) == {"looker_group_label": "Order Info"} + + def test_entity_config_meta_becomes_custom_extension(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + entities=[ + _entity( + "order_id", + entity_type=EntityType.PRIMARY, + config=PydanticSemanticLayerElementConfig(meta={"hidden": True}), + ) + ], + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + extensions = _fields(result)[0].custom_extensions + assert extensions is not None + assert json.loads(extensions[0].data) == {"hidden": True} + + def test_measure_config_meta_becomes_custom_extension(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + measures=[ + _measure( + "revenue", + config=PydanticSemanticLayerElementConfig(meta={"value_format": "$#,##0.00"}), + ) + ], + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + extensions = _fields(result)[0].custom_extensions + assert extensions is not None + assert json.loads(extensions[0].data) == {"value_format": "$#,##0.00"} + + def test_metric_config_meta_becomes_custom_extension(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + measures=[_measure("revenue", agg=AggregationType.SUM, expr="amount")], + ) + revenue = _simple_metric( + "revenue", + "revenue", + config=PydanticSemanticLayerElementConfig(meta={"group_label": "Revenue"}), + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[revenue])).output + + extensions = _ossie_metrics(result)[0].custom_extensions + assert extensions is not None + assert json.loads(extensions[0].data) == {"group_label": "Revenue"} + + def test_semantic_model_config_meta_becomes_dataset_custom_extension(self) -> None: + sm = PydanticSemanticModel( + name="orders", + defaults=None, + description=None, + node_relation=PydanticNodeRelation(schema_name="analytics", alias="orders_table"), + primary_entity=None, + metadata=default_meta(), + config=PydanticSemanticLayerElementConfig(meta={"view_label": "Orders"}), + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + extensions = result.semantic_model[0].datasets[0].custom_extensions + assert extensions is not None + assert json.loads(extensions[0].data) == {"view_label": "Orders"} + + def test_no_config_meta_omits_custom_extensions(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + dimensions=[_dimension("status")], + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + assert _fields(result)[0].custom_extensions is None + + def test_empty_config_meta_omits_custom_extensions(self) -> None: + sm = semantic_model_with_guaranteed_meta( + name="orders", + dimensions=[_dimension("status", config=PydanticSemanticLayerElementConfig(meta={}))], + ) + result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output + + assert _fields(result)[0].custom_extensions is None + + class TestEntityKeyExtraction: @pytest.mark.parametrize( "entity_type, name, expr, expected_pk, expected_uk", diff --git a/converters/dbt/tests/test_ossie_to_msi.py b/converters/dbt/tests/test_ossie_to_msi.py index eaec212b..4fd6a10e 100644 --- a/converters/dbt/tests/test_ossie_to_msi.py +++ b/converters/dbt/tests/test_ossie_to_msi.py @@ -17,10 +17,12 @@ """Tests for OssieToMSIConverter.""" +import json + import pytest from syrupy.assertion import SnapshotAssertion -from ossie import OssieDataType, OssieDimension +from ossie import OssieCustomExtension, OssieDataType, OssieDimension, OssieVendor from ossie_dbt.msi_to_ossie import MSIToOssieConverter from ossie_dbt.ossie_to_msi import OssieToMSIConverter from metricflow_semantic_interfaces.type_enums import ( @@ -406,6 +408,84 @@ def test_percentile_cont_non_median_carries_percentile_param(self) -> None: assert m.type_params.expr == "amount" +class TestOssieToMSIConfigMetaRoundTrip: + """The DBT custom_extensions entry this converter's forward direction writes (ossie#303) + round-trips back to config.meta; extensions from another vendor do not.""" + + def test_field_custom_extension_becomes_entity_config_meta(self) -> None: + ext = [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps({"group_label": "IDs"}))] + doc = _ossie_doc( + datasets=[ + _ossie_dataset( + "orders", + fields=[_ossie_field("order_id", custom_extensions=ext)], + primary_key=["order_id"], + ) + ] + ) + sm = OssieToMSIConverter().convert(doc).output.semantic_models[0] + + assert sm.entities[0].config is not None + assert sm.entities[0].config.meta == {"group_label": "IDs"} + + def test_field_custom_extension_becomes_dimension_config_meta(self) -> None: + ext = [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps({"value_format": "$#,##0"}))] + doc = _ossie_doc( + datasets=[_ossie_dataset("orders", fields=[_ossie_field("amount", custom_extensions=ext)])] + ) + sm = OssieToMSIConverter().convert(doc).output.semantic_models[0] + + assert sm.dimensions[0].config is not None + assert sm.dimensions[0].config.meta == {"value_format": "$#,##0"} + + def test_dataset_custom_extension_becomes_semantic_model_config_meta(self) -> None: + ext = [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps({"view_label": "Orders"}))] + doc = _ossie_doc(datasets=[_ossie_dataset("orders", custom_extensions=ext)]) + sm = OssieToMSIConverter().convert(doc).output.semantic_models[0] + + assert sm.config is not None + assert sm.config.meta == {"view_label": "Orders"} + + def test_metric_custom_extension_becomes_metric_config_meta(self) -> None: + ext = [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps({"group_label": "Revenue"}))] + doc = _ossie_doc( + datasets=[_ossie_dataset("orders", fields=[_ossie_field("amount")])], + metrics=[_ossie_metric("revenue", "SUM(amount)", custom_extensions=ext)], + ) + result = OssieToMSIConverter().convert(doc).output + + assert result.metrics[0].config is not None + assert result.metrics[0].config.meta == {"group_label": "Revenue"} + + def test_ratio_sub_metrics_do_not_inherit_parent_config_meta(self) -> None: + """Numerator/denominator are synthetic metrics with no origin of their own.""" + ext = [OssieCustomExtension(vendor_name=OssieVendor.DBT.value, data=json.dumps({"group_label": "ARPU"}))] + doc = _ossie_doc( + datasets=[_ossie_dataset("orders", fields=[_ossie_field("amount"), _ossie_field("order_id")])], + metrics=[_ossie_metric("arpu", "(SUM(amount)) / (COUNT(order_id))", custom_extensions=ext)], + ) + result = OssieToMSIConverter().convert(doc).output + + ratio = next(m for m in result.metrics if m.type == MetricType.RATIO) + assert ratio.config is not None + assert ratio.config.meta == {"group_label": "ARPU"} + sub_metrics = [m for m in result.metrics if m.type == MetricType.SIMPLE] + assert all(m.config is None for m in sub_metrics) + + def test_non_dbt_vendor_extension_does_not_become_config_meta(self) -> None: + ext = [OssieCustomExtension(vendor_name=OssieVendor.SIGMA.value, data=json.dumps({"foo": "bar"}))] + doc = _ossie_doc(datasets=[_ossie_dataset("orders", fields=[_ossie_field("status", custom_extensions=ext)])]) + sm = OssieToMSIConverter().convert(doc).output.semantic_models[0] + + assert sm.dimensions[0].config is None + + def test_no_custom_extensions_produces_no_config(self) -> None: + doc = _ossie_doc(datasets=[_ossie_dataset("orders", fields=[_ossie_field("status")])]) + sm = OssieToMSIConverter().convert(doc).output.semantic_models[0] + + assert sm.dimensions[0].config is None + + class TestOssieToMSIRoundTrip: def test_ossie_to_msi_to_ossie_preserves_structure(self, snapshot: SnapshotAssertion) -> None: """Ossie → MSI → Ossie preserves dataset names, fields, and metric expressions."""