From e65f7d219a49b9159bfcdda46cdf26ee68e5451b Mon Sep 17 00:00:00 2001 From: Ashfaq <105435085+Ashfaqbs@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:15:53 +0530 Subject: [PATCH 1/2] fix(salesforce): export OSI metrics as semanticCalculatedMeasurements MetricMappingHandler.mapOssieToSalesforce() was a stub that discarded every metric mapping and returned without writing anything, so an Ossie model's metrics never appeared in the exported Salesforce semanticCalculatedMeasurements. Wire up the existing generic name/description mapping (mappings.yaml already declared metrics.name/metrics.description, unused until now), and add unwrapExpressions() to flatten each metric's expression into a Salesforce-compatible string. TABLEAU is preferred when present since that is what Salesforce/Tableau CRM speaks; a model authored without one falls back to its ANSI_SQL expression unresolved/untranslated, since actually resolving/rewriting into TABLEAU syntax depends on #222's still-open expression-language work, not this fix. A metric with neither dialect fails the conversion with an actionable error naming the metric, rather than being silently dropped. Fixes #399 --- .../ossie/converter/ConverterConstants.java | 1 + .../ossie/converter/MetricMappingHandler.java | 71 +++++++++++++++- .../ossie/OssieToSalesforceConverterTest.java | 82 ++++++++++++++++++- 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java index 044dfe4c..02c0bb26 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java @@ -89,6 +89,7 @@ public enum Level { public static final String DIALECTS = "dialects"; public static final String DIALECT = "dialect"; public static final String DIALECT_TABLEAU = "TABLEAU"; + public static final String DIALECT_ANSI_SQL = "ANSI_SQL"; // Relationship properties public static final String CRITERIA = "criteria"; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 6add2b9b..00de289d 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -24,6 +24,7 @@ import org.apache.ossie.converter.ConverterConstants.Level; import org.apache.ossie.converter.pipeline.PipelineStep; +import org.apache.ossie.exception.ConversionException; import java.util.*; import org.apache.ossie.util.MappingUtils; @@ -78,9 +79,16 @@ private void mapOssieToSalesforce( // Filter mappings to get only metric-related entries Map metricMappings = MappingUtils.filterMappingsByPrefix(mappings, METRICS); + + Map mappedData = GenericMappingEngine.applyMappings(sourceData, metricMappings); metricMappings.keySet().forEach(mappings::remove); - logger.debug("Metrics are not mapped in Ossie to Salesforce direction"); + outputData.putAll(mappedData); + + List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); + if (sfMetrics != null) { + unwrapExpressions(ossieMetrics, sfMetrics); + } } /** @@ -118,6 +126,67 @@ private void mapSalesforceToOssie( } + /** + * Unwraps expressions for Ossie→SF conversion, mirroring {@link #wrapExpressions}. + * + *

Picks an expression out of each Ossie metric's {@code expression.dialects[]} and + * flattens it into the Salesforce metric's {@code expression} string. {@code TABLEAU} is + * preferred (it is what Salesforce/Tableau CRM itself speaks); a model authored without one + * falls back to {@code ANSI_SQL} best-effort, since resolving/rewriting an expression into + * TABLEAU syntax is the scope of #222's expression-language work, not this fix. A metric with + * neither dialect fails the conversion rather than being silently omitted (#399). + */ + private void unwrapExpressions(List ossieMetrics, List sfMetrics) { + for (int i = 0; i < ossieMetrics.size() && i < sfMetrics.size(); i++) { + Map ossieMetric = asMap(ossieMetrics.get(i)); + Map sfMetric = asMap(sfMetrics.get(i)); + + String expressionValue = extractExpression(ossieMetric, DIALECT_TABLEAU); + if (expressionValue == null) { + expressionValue = extractExpression(ossieMetric, DIALECT_ANSI_SQL); + if (expressionValue != null) { + logger.warn( + "Metric '{}' has no TABLEAU-dialect expression; exporting its " + + "ANSI_SQL expression to Salesforce unresolved/untranslated", + getString(ossieMetric, NAME)); + } + } + if (expressionValue == null) { + throw new ConversionException( + "Metric '" + getString(ossieMetric, NAME) + "' has neither a TABLEAU nor " + + "an ANSI_SQL expression to export to Salesforce; add one to " + + "expression.dialects[] or remove the metric."); + } + sfMetric.put(EXPRESSION, expressionValue); + + String datatype = SalesforceDataTypeMapper.toSalesforce(getString(ossieMetric, OSSIE_DATATYPE)); + if (datatype != null) { + sfMetric.put(DATA_TYPE, datatype); + } + } + } + + /** + * Finds the given dialect's expression string in an Ossie metric's + * {@code expression.dialects[]}, or {@code null} when the metric has no expression or no + * entry for that dialect. + */ + private String extractExpression(Map ossieMetric, String dialect) { + Map expression = getMap(ossieMetric, EXPRESSION); + if (expression == null) { + return null; + } + List dialects = getList(expression, DIALECTS); + if (dialects == null) { + return null; + } + return streamMaps(dialects) + .filter(d -> dialect.equals(getString(d, DIALECT))) + .map(d -> getString(d, EXPRESSION)) + .findFirst() + .orElse(null); + } + /** * Wraps expressions for SF→Ossie conversion. */ diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 6d5c1237..627637f0 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -287,12 +287,90 @@ void testCustomExtensionsRestoration() throws Exception { } @Test - void testMetricsNotConvertedInOssieToSalesforce() throws Exception { + void testMetricsConvertedToSemanticCalculatedMeasurements() throws Exception { List results = converter.convert(ossieYaml); Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); - assertNull(calcMeasurements, "Metrics from Ossie are not converted to semanticCalculatedMeasurements in Ossie->SF direction"); + assertNotNull(calcMeasurements, "Metrics from Ossie should convert to semanticCalculatedMeasurements"); + assertEquals(2, calcMeasurements.size()); + + Map totalRevenue = calcMeasurements.stream() + .filter(m -> "total_revenue".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(totalRevenue); + assertEquals("Sum of all order amounts", totalRevenue.get("description")); + assertEquals("Number", totalRevenue.get("dataType")); + // The fixture's metrics only carry an ANSI_SQL dialect (no TABLEAU) -- falls back to + // exporting it unresolved/untranslated rather than failing the whole conversion. + assertEquals("SUM([Orders].[amount])", totalRevenue.get("expression")); + + Map avgOrderValue = calcMeasurements.stream() + .filter(m -> "avg_order_value".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(avgOrderValue); + assertEquals("AVG([Orders].[amount])", avgOrderValue.get("expression")); + } + + @Test + void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { + // Normalize line endings first: the fixture file may check out with CRLF depending on + // the platform's autocrlf setting, but the substitution below is written with LF. + String yamlWithTableauMetric = ossieYaml.replace("\r\n", "\n").replace( + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n", + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n" + + " - dialect: TABLEAU\n" + + " expression: SUM(Orders.amount)\n"); + assertTrue(yamlWithTableauMetric.contains("dialect: TABLEAU"), "fixture text substitution did not match"); + + List results = converter.convert(yamlWithTableauMetric); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); + + Map totalRevenue = calcMeasurements.stream() + .filter(m -> "total_revenue".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(totalRevenue); + assertEquals("SUM(Orders.amount)", totalRevenue.get("expression"), + "TABLEAU dialect should be preferred over ANSI_SQL when both are present"); + } + + @Test + void testMetricWithNoConvertibleDialectFailsConversion() throws Exception { + // Normalize line endings first: the fixture file may check out with CRLF depending on + // the platform's autocrlf setting, but the substitution below is written with LF. + // The Ossie schema requires every metric to have an expression and restricts `dialect` + // to its own enum, so this uses BIGQUERY (a valid dialect, but neither TABLEAU nor + // ANSI_SQL) rather than omitting the expression or inventing an unrecognized dialect. + String yamlWithUnconvertibleDialect = ossieYaml.replace("\r\n", "\n").replace( + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n", + " - dialect: BIGQUERY\n" + + " expression: SUM(Orders.amount)\n"); + assertTrue(yamlWithUnconvertibleDialect.contains("dialect: BIGQUERY"), + "fixture text substitution did not match"); + + Exception exception = + assertThrows(Exception.class, () -> converter.convert(yamlWithUnconvertibleDialect)); + String message = exception.getMessage() != null ? exception.getMessage() : exception.getCause().getMessage(); + assertTrue(message.contains("total_revenue"), "error should name the unconvertible metric: " + message); } @Test From e618f9d88c0374e4bd0627cdbdd3677c6ae698ec Mon Sep 17 00:00:00 2001 From: Ashfaq <105435085+Ashfaqbs@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:53:48 +0530 Subject: [PATCH 2/2] fix(salesforce): restore metric custom_extensions, enforce TABLEAU-only expressions Addresses jbonofre's review on #402: - Restore custom_extensions at Level.METRICS before unwrapExpressions() in the Ossie->Salesforce direction, so label, dataType and other SALESFORCE-vendor fields round-trip instead of being silently dropped. Default label to apiName when nothing was restored. - unwrapExpressions() now uses putIfAbsent for the mapped dataType, so an exact value restored from custom_extensions (e.g. Currency) is not overwritten by the generic Decimal->Number mapping. - Drop the ANSI_SQL fallback in unwrapExpressions(): Salesforce's expression field is Tableau syntax, so an untranslated ANSI_SQL expression would deploy wrong or fail. A metric now needs a TABLEAU dialect to export, matching the TABLEAU-only scope from #399 and FieldMappingHandler's existing behavior. Relabeled the two metrics in the shared test fixture that were marked ANSI_SQL but already used Tableau bracket syntax (SUM([Orders].[amount])). - Tightened testMetricWithNoConvertibleDialectFailsConversion to assert the exact ConversionException instead of a generic Exception whose getCause() could NPE. - Added a regression test covering metric custom_extensions restore (label + exact dataType) and the apiName-default fallback. --- .../ossie/converter/MetricMappingHandler.java | 49 ++- .../main/resources/schemas/ossie-schema.json | 352 ++++++++++++++++++ .../ossie/OssieToSalesforceConverterTest.java | 78 +++- .../resources/examples/ossieToSalesforce.yaml | 4 +- 4 files changed, 449 insertions(+), 34 deletions(-) create mode 100644 converters/salesforce/src/main/resources/schemas/ossie-schema.json diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 00de289d..542219cd 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -85,9 +85,27 @@ private void mapOssieToSalesforce( outputData.putAll(mappedData); + // Restore before unwrapExpressions(): restore only fills absent keys, so exact + // Salesforce values (e.g. dataType Currency) must land before derived ones. + customExtensionHandler.restoreCustomExtensionsAtLevel(outputData, sourceData, Level.METRICS); + List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); if (sfMetrics != null) { unwrapExpressions(ossieMetrics, sfMetrics); + applyDefaults(sfMetrics); + } + } + + /** + * Applies default values for required Salesforce calculated measurement properties. + * Used when converting Ossie → Salesforce. + */ + private void applyDefaults(List sfMetrics) { + for (Object sfMetricObj : sfMetrics) { + Map sfMetric = asMap(sfMetricObj); + if (!sfMetric.containsKey(LABEL) && sfMetric.containsKey(API_NAME)) { + sfMetric.put(LABEL, getString(sfMetric, API_NAME)); + } } } @@ -129,12 +147,12 @@ private void mapSalesforceToOssie( /** * Unwraps expressions for Ossie→SF conversion, mirroring {@link #wrapExpressions}. * - *

Picks an expression out of each Ossie metric's {@code expression.dialects[]} and - * flattens it into the Salesforce metric's {@code expression} string. {@code TABLEAU} is - * preferred (it is what Salesforce/Tableau CRM itself speaks); a model authored without one - * falls back to {@code ANSI_SQL} best-effort, since resolving/rewriting an expression into - * TABLEAU syntax is the scope of #222's expression-language work, not this fix. A metric with - * neither dialect fails the conversion rather than being silently omitted (#399). + *

Picks the {@code TABLEAU}-dialect expression out of each Ossie metric's + * {@code expression.dialects[]} and flattens it into the Salesforce metric's + * {@code expression} string. {@code TABLEAU} is what Salesforce/Tableau CRM itself speaks; + * an {@code ANSI_SQL} expression is not translated to Tableau syntax, since that is the scope + * of #222's expression-language work, not this fix (see #403). A metric with no TABLEAU + * dialect fails the conversion rather than exporting an untranslated expression (#399). */ private void unwrapExpressions(List ossieMetrics, List sfMetrics) { for (int i = 0; i < ossieMetrics.size() && i < sfMetrics.size(); i++) { @@ -142,26 +160,19 @@ private void unwrapExpressions(List ossieMetrics, List sfMetrics Map sfMetric = asMap(sfMetrics.get(i)); String expressionValue = extractExpression(ossieMetric, DIALECT_TABLEAU); - if (expressionValue == null) { - expressionValue = extractExpression(ossieMetric, DIALECT_ANSI_SQL); - if (expressionValue != null) { - logger.warn( - "Metric '{}' has no TABLEAU-dialect expression; exporting its " - + "ANSI_SQL expression to Salesforce unresolved/untranslated", - getString(ossieMetric, NAME)); - } - } if (expressionValue == null) { throw new ConversionException( - "Metric '" + getString(ossieMetric, NAME) + "' has neither a TABLEAU nor " - + "an ANSI_SQL expression to export to Salesforce; add one to " - + "expression.dialects[] or remove the metric."); + "Metric '" + getString(ossieMetric, NAME) + "' has no TABLEAU expression " + + "to export to Salesforce; add one to expression.dialects[] or " + + "remove the metric."); } sfMetric.put(EXPRESSION, expressionValue); String datatype = SalesforceDataTypeMapper.toSalesforce(getString(ossieMetric, OSSIE_DATATYPE)); if (datatype != null) { - sfMetric.put(DATA_TYPE, datatype); + // Exact Salesforce value restored from custom_extensions wins over the + // mapped Ossie datatype (e.g. a restored "Currency" over a mapped "Decimal"). + sfMetric.putIfAbsent(DATA_TYPE, datatype); } } } diff --git a/converters/salesforce/src/main/resources/schemas/ossie-schema.json b/converters/salesforce/src/main/resources/schemas/ossie-schema.json new file mode 100644 index 00000000..f20f0e92 --- /dev/null +++ b/converters/salesforce/src/main/resources/schemas/ossie-schema.json @@ -0,0 +1,352 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/ossie-schema.json", + "title": "Apache Ossie Core Metadata Specification", + "description": "JSON Schema for validating Apache Ossie semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Apache Ossie specification version" + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], + "description": "Vendor name for custom extensions. Any string value is accepted." + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "DataType": { + "type": "string", + "enum": [ + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", + "Opaque" + ], + "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown; use `Opaque` plus `custom_extensions` for a known type outside the portable vocabulary." + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment." + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 627637f0..7cc26b7e 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -25,6 +25,7 @@ import org.apache.ossie.converter.ConverterFactory; import org.apache.ossie.converter.ConversionDirection; import org.apache.ossie.converter.CustomExtensionHandler; +import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -302,8 +303,6 @@ void testMetricsConvertedToSemanticCalculatedMeasurements() throws Exception { assertNotNull(totalRevenue); assertEquals("Sum of all order amounts", totalRevenue.get("description")); assertEquals("Number", totalRevenue.get("dataType")); - // The fixture's metrics only carry an ANSI_SQL dialect (no TABLEAU) -- falls back to - // exporting it unresolved/untranslated rather than failing the whole conversion. assertEquals("SUM([Orders].[amount])", totalRevenue.get("expression")); Map avgOrderValue = calcMeasurements.stream() @@ -325,7 +324,7 @@ void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { + " datatype: Decimal\n" + " expression:\n" + " dialects:\n" - + " - dialect: ANSI_SQL\n" + + " - dialect: TABLEAU\n" + " expression: SUM([Orders].[amount])\n", " metrics:\n" + " - description: Sum of all order amounts\n" @@ -334,9 +333,9 @@ void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" - + " expression: SUM([Orders].[amount])\n" + + " expression: SUM(Orders.amount)\n" + " - dialect: TABLEAU\n" - + " expression: SUM(Orders.amount)\n"); + + " expression: SUM([Orders].[amount])\n"); assertTrue(yamlWithTableauMetric.contains("dialect: TABLEAU"), "fixture text substitution did not match"); List results = converter.convert(yamlWithTableauMetric); @@ -348,7 +347,7 @@ void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { .findFirst() .orElse(null); assertNotNull(totalRevenue); - assertEquals("SUM(Orders.amount)", totalRevenue.get("expression"), + assertEquals("SUM([Orders].[amount])", totalRevenue.get("expression"), "TABLEAU dialect should be preferred over ANSI_SQL when both are present"); } @@ -357,20 +356,73 @@ void testMetricWithNoConvertibleDialectFailsConversion() throws Exception { // Normalize line endings first: the fixture file may check out with CRLF depending on // the platform's autocrlf setting, but the substitution below is written with LF. // The Ossie schema requires every metric to have an expression and restricts `dialect` - // to its own enum, so this uses BIGQUERY (a valid dialect, but neither TABLEAU nor - // ANSI_SQL) rather than omitting the expression or inventing an unrecognized dialect. + // to its own enum, so this uses BIGQUERY (a valid dialect, but not TABLEAU) rather than + // omitting the expression or inventing an unrecognized dialect. String yamlWithUnconvertibleDialect = ossieYaml.replace("\r\n", "\n").replace( - " - dialect: ANSI_SQL\n" + " - dialect: TABLEAU\n" + " expression: SUM([Orders].[amount])\n", " - dialect: BIGQUERY\n" + " expression: SUM(Orders.amount)\n"); assertTrue(yamlWithUnconvertibleDialect.contains("dialect: BIGQUERY"), "fixture text substitution did not match"); - Exception exception = - assertThrows(Exception.class, () -> converter.convert(yamlWithUnconvertibleDialect)); - String message = exception.getMessage() != null ? exception.getMessage() : exception.getCause().getMessage(); - assertTrue(message.contains("total_revenue"), "error should name the unconvertible metric: " + message); + ConversionException exception = + assertThrows(ConversionException.class, () -> converter.convert(yamlWithUnconvertibleDialect)); + assertTrue(exception.getMessage().contains("total_revenue"), + "error should name the unconvertible metric: " + exception.getMessage()); + } + + @Test + void testMetricCustomExtensionsRestoredBeforeUnwrap() throws Exception { + // Normalize line endings first: the fixture file may check out with CRLF depending on + // the platform's autocrlf setting, but the substitution below is written with LF. + String yamlWithMetricExtensions = ossieYaml.replace("\r\n", "\n").replace( + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: TABLEAU\n" + + " expression: SUM([Orders].[amount])\n", + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: TABLEAU\n" + + " expression: SUM([Orders].[amount])\n" + + " custom_extensions:\n" + + " - vendor_name: SALESFORCE\n" + + " data: |-\n" + + " {\n" + + " \"label\" : \"Total Revenue\",\n" + + " \"dataType\" : \"Currency\"\n" + + " }\n"); + assertTrue(yamlWithMetricExtensions.contains("Total Revenue"), "fixture text substitution did not match"); + + List results = converter.convert(yamlWithMetricExtensions); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); + + Map totalRevenue = calcMeasurements.stream() + .filter(m -> "total_revenue".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(totalRevenue); + assertEquals("Total Revenue", totalRevenue.get("label"), + "label restored from custom_extensions should win over the apiName default"); + assertEquals("Currency", totalRevenue.get("dataType"), + "exact Salesforce dataType restored from custom_extensions should win over the mapped Decimal->Number"); + + Map avgOrderValue = calcMeasurements.stream() + .filter(m -> "avg_order_value".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(avgOrderValue); + assertEquals("avg_order_value", avgOrderValue.get("label"), + "metrics with no custom_extensions should default label to apiName"); } @Test diff --git a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml index 101588f0..c9926279 100644 --- a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml +++ b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml @@ -392,12 +392,12 @@ semantic_model: datatype: Decimal expression: dialects: - - dialect: ANSI_SQL + - dialect: TABLEAU expression: SUM([Orders].[amount]) - description: Average amount per order name: avg_order_value datatype: Decimal expression: dialects: - - dialect: ANSI_SQL + - dialect: TABLEAU expression: AVG([Orders].[amount])