From e65f7d219a49b9159bfcdda46cdf26ee68e5451b Mon Sep 17 00:00:00 2001 From: Ashfaq <105435085+Ashfaqbs@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:15:53 +0530 Subject: [PATCH 1/6] fix(salesforce): export OSI metrics as semanticCalculatedMeasurements MetricMappingHandler.mapOssieToSalesforce() was a stub that discarded every metric mapping and returned without writing anything, so an Ossie model's metrics never appeared in the exported Salesforce semanticCalculatedMeasurements. Wire up the existing generic name/description mapping (mappings.yaml already declared metrics.name/metrics.description, unused until now), and add unwrapExpressions() to flatten each metric's expression into a Salesforce-compatible string. TABLEAU is preferred when present since that is what Salesforce/Tableau CRM speaks; a model authored without one falls back to its ANSI_SQL expression unresolved/untranslated, since actually resolving/rewriting into TABLEAU syntax depends on #222's still-open expression-language work, not this fix. A metric with neither dialect fails the conversion with an actionable error naming the metric, rather than being silently dropped. Fixes #399 --- .../ossie/converter/ConverterConstants.java | 1 + .../ossie/converter/MetricMappingHandler.java | 71 +++++++++++++++- .../ossie/OssieToSalesforceConverterTest.java | 82 ++++++++++++++++++- 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java index 044dfe4c..02c0bb26 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java @@ -89,6 +89,7 @@ public enum Level { public static final String DIALECTS = "dialects"; public static final String DIALECT = "dialect"; public static final String DIALECT_TABLEAU = "TABLEAU"; + public static final String DIALECT_ANSI_SQL = "ANSI_SQL"; // Relationship properties public static final String CRITERIA = "criteria"; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 6add2b9b..00de289d 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -24,6 +24,7 @@ import org.apache.ossie.converter.ConverterConstants.Level; import org.apache.ossie.converter.pipeline.PipelineStep; +import org.apache.ossie.exception.ConversionException; import java.util.*; import org.apache.ossie.util.MappingUtils; @@ -78,9 +79,16 @@ private void mapOssieToSalesforce( // Filter mappings to get only metric-related entries Map metricMappings = MappingUtils.filterMappingsByPrefix(mappings, METRICS); + + Map mappedData = GenericMappingEngine.applyMappings(sourceData, metricMappings); metricMappings.keySet().forEach(mappings::remove); - logger.debug("Metrics are not mapped in Ossie to Salesforce direction"); + outputData.putAll(mappedData); + + List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); + if (sfMetrics != null) { + unwrapExpressions(ossieMetrics, sfMetrics); + } } /** @@ -118,6 +126,67 @@ private void mapSalesforceToOssie( } + /** + * Unwraps expressions for Ossie→SF conversion, mirroring {@link #wrapExpressions}. + * + *

Picks an expression out of each Ossie metric's {@code expression.dialects[]} and + * flattens it into the Salesforce metric's {@code expression} string. {@code TABLEAU} is + * preferred (it is what Salesforce/Tableau CRM itself speaks); a model authored without one + * falls back to {@code ANSI_SQL} best-effort, since resolving/rewriting an expression into + * TABLEAU syntax is the scope of #222's expression-language work, not this fix. A metric with + * neither dialect fails the conversion rather than being silently omitted (#399). + */ + private void unwrapExpressions(List ossieMetrics, List sfMetrics) { + for (int i = 0; i < ossieMetrics.size() && i < sfMetrics.size(); i++) { + Map ossieMetric = asMap(ossieMetrics.get(i)); + Map sfMetric = asMap(sfMetrics.get(i)); + + String expressionValue = extractExpression(ossieMetric, DIALECT_TABLEAU); + if (expressionValue == null) { + expressionValue = extractExpression(ossieMetric, DIALECT_ANSI_SQL); + if (expressionValue != null) { + logger.warn( + "Metric '{}' has no TABLEAU-dialect expression; exporting its " + + "ANSI_SQL expression to Salesforce unresolved/untranslated", + getString(ossieMetric, NAME)); + } + } + if (expressionValue == null) { + throw new ConversionException( + "Metric '" + getString(ossieMetric, NAME) + "' has neither a TABLEAU nor " + + "an ANSI_SQL expression to export to Salesforce; add one to " + + "expression.dialects[] or remove the metric."); + } + sfMetric.put(EXPRESSION, expressionValue); + + String datatype = SalesforceDataTypeMapper.toSalesforce(getString(ossieMetric, OSSIE_DATATYPE)); + if (datatype != null) { + sfMetric.put(DATA_TYPE, datatype); + } + } + } + + /** + * Finds the given dialect's expression string in an Ossie metric's + * {@code expression.dialects[]}, or {@code null} when the metric has no expression or no + * entry for that dialect. + */ + private String extractExpression(Map ossieMetric, String dialect) { + Map expression = getMap(ossieMetric, EXPRESSION); + if (expression == null) { + return null; + } + List dialects = getList(expression, DIALECTS); + if (dialects == null) { + return null; + } + return streamMaps(dialects) + .filter(d -> dialect.equals(getString(d, DIALECT))) + .map(d -> getString(d, EXPRESSION)) + .findFirst() + .orElse(null); + } + /** * Wraps expressions for SF→Ossie conversion. */ diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 6d5c1237..627637f0 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -287,12 +287,90 @@ void testCustomExtensionsRestoration() throws Exception { } @Test - void testMetricsNotConvertedInOssieToSalesforce() throws Exception { + void testMetricsConvertedToSemanticCalculatedMeasurements() throws Exception { List results = converter.convert(ossieYaml); Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); - assertNull(calcMeasurements, "Metrics from Ossie are not converted to semanticCalculatedMeasurements in Ossie->SF direction"); + assertNotNull(calcMeasurements, "Metrics from Ossie should convert to semanticCalculatedMeasurements"); + assertEquals(2, calcMeasurements.size()); + + Map totalRevenue = calcMeasurements.stream() + .filter(m -> "total_revenue".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(totalRevenue); + assertEquals("Sum of all order amounts", totalRevenue.get("description")); + assertEquals("Number", totalRevenue.get("dataType")); + // The fixture's metrics only carry an ANSI_SQL dialect (no TABLEAU) -- falls back to + // exporting it unresolved/untranslated rather than failing the whole conversion. + assertEquals("SUM([Orders].[amount])", totalRevenue.get("expression")); + + Map avgOrderValue = calcMeasurements.stream() + .filter(m -> "avg_order_value".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(avgOrderValue); + assertEquals("AVG([Orders].[amount])", avgOrderValue.get("expression")); + } + + @Test + void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { + // Normalize line endings first: the fixture file may check out with CRLF depending on + // the platform's autocrlf setting, but the substitution below is written with LF. + String yamlWithTableauMetric = ossieYaml.replace("\r\n", "\n").replace( + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n", + " metrics:\n" + + " - description: Sum of all order amounts\n" + + " name: total_revenue\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n" + + " - dialect: TABLEAU\n" + + " expression: SUM(Orders.amount)\n"); + assertTrue(yamlWithTableauMetric.contains("dialect: TABLEAU"), "fixture text substitution did not match"); + + List results = converter.convert(yamlWithTableauMetric); + Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); + List> calcMeasurements = (List>) sfModel.get("semanticCalculatedMeasurements"); + + Map totalRevenue = calcMeasurements.stream() + .filter(m -> "total_revenue".equals(m.get("apiName"))) + .findFirst() + .orElse(null); + assertNotNull(totalRevenue); + assertEquals("SUM(Orders.amount)", totalRevenue.get("expression"), + "TABLEAU dialect should be preferred over ANSI_SQL when both are present"); + } + + @Test + void testMetricWithNoConvertibleDialectFailsConversion() throws Exception { + // Normalize line endings first: the fixture file may check out with CRLF depending on + // the platform's autocrlf setting, but the substitution below is written with LF. + // The Ossie schema requires every metric to have an expression and restricts `dialect` + // to its own enum, so this uses BIGQUERY (a valid dialect, but neither TABLEAU nor + // ANSI_SQL) rather than omitting the expression or inventing an unrecognized dialect. + String yamlWithUnconvertibleDialect = ossieYaml.replace("\r\n", "\n").replace( + " - dialect: ANSI_SQL\n" + + " expression: SUM([Orders].[amount])\n", + " - dialect: BIGQUERY\n" + + " expression: SUM(Orders.amount)\n"); + assertTrue(yamlWithUnconvertibleDialect.contains("dialect: BIGQUERY"), + "fixture text substitution did not match"); + + Exception exception = + assertThrows(Exception.class, () -> converter.convert(yamlWithUnconvertibleDialect)); + String message = exception.getMessage() != null ? exception.getMessage() : exception.getCause().getMessage(); + assertTrue(message.contains("total_revenue"), "error should name the unconvertible metric: " + message); } @Test From 2ff68e87408de5a6e4b2d0e09a9d22e6b1318e40 Mon Sep 17 00:00:00 2001 From: Saurabh Deshpande <43935865+saurabhdeshp@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:52:48 -0700 Subject: [PATCH 2/6] fix(salesforce): translate metric expressions and validate references --- converters/salesforce/README.md | 128 +++- .../ossie/app/OssieSalesforceConverter.java | 1 + .../converter/MetricExpressionTranslator.java | 563 ++++++++++++++++++ .../ossie/converter/MetricFieldResolver.java | 230 +++++++ .../ossie/converter/MetricMappingHandler.java | 82 +-- .../java/org/apache/ossie/MetricCliTest.java | 60 ++ .../ossie/MetricExportIntegrationTest.java | 386 ++++++++++++ .../ossie/OssieToSalesforceConverterTest.java | 9 +- .../MetricExpressionSemanticsTest.java | 437 ++++++++++++++ .../MetricExpressionTranslatorTest.java | 276 +++++++++ .../converter/MetricFieldResolverTest.java | 235 ++++++++ 11 files changed, 2342 insertions(+), 65 deletions(-) create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md index 3b11559c..7130165f 100644 --- a/converters/salesforce/README.md +++ b/converters/salesforce/README.md @@ -42,7 +42,8 @@ This produces a self-contained executable jar at `target/ossie-salesforce-conver ## Setup -Both schemas must be obtained and placed under `src/main/resources/schemas/` before building, so they get bundled into the jar. +Obtain the Salesforce schema before building so it is bundled into the jar. +Maven copies the canonical Ossie schema from `../../core-spec/ossie-schema.json`. ### Salesforce Semantic Model Schema @@ -50,11 +51,15 @@ Both schemas must be obtained and placed under `src/main/resources/schemas/` bef 2. Copy the JSON schema content from the page 3. Save it to `src/main/resources/schemas/salesforce-semantic-model-schema.json` -### Apache Ossie Schema +Run the complete suite, including Salesforce schema checks, with: -1. Visit the [Ossie schema on GitHub](https://github.com/apache/ossie/blob/main/core-spec/ossie-schema.json) -2. Copy the raw JSON contents -3. Save it to `src/main/resources/schemas/ossie-schema.json` +```bash +mvn -DrequireSalesforceSchema=true clean verify +``` + +The property makes a missing Salesforce schema fail the test run. Without it, +schema-dependent tests retain their existing skip behavior. `verify` also checks +Apache license headers. Do not commit downloaded schemas. ## Usage @@ -168,7 +173,7 @@ ossieToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); | Field `datatype` | Field `dataType` when a safe mapping exists | | `relationships[]` | `semanticRelationships[]` | | `from_columns` + `to_columns` | `criteria[]` | -| `metrics[]` | Not currently exported | +| `metrics[]` | Validated Tua expressions in `semanticCalculatedMeasurements[]` | | `ai_context` | `businessPreferences` | | `custom_extensions` (vendor: `SALESFORCE`) | Restored properties | @@ -229,6 +234,117 @@ dimensions. **Unsupported relationships** (containing Formula or SemanticField types) are stored in `custom_extensions` at the model level rather than being converted to Ossie relationships. +### Metric expressions + +Metrics select `TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`, independent of entry +order. The selected expression is parsed and validated; an invalid preferred +expression fails rather than falling back to another dialect. Duplicate selected +dialect entries are errors. Successful conversion exports every declared metric. + +The target is the Salesforce/Tableau Next semantic model's +[Tua calculation language](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-functions.html). +Calculated measurements emit `syntax: Tua`, `dataType: Number`, and +`aggregationType: UserAgg`, so an already aggregated formula is not aggregated +again. See [calculated fields](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-calculated-fields.html) +and [aggregation rules](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-aggregation.html). + +| SQL input | Tua output | +|-----------|------------| +| `SUM`, `AVG`, `MIN`, `MAX`, `COUNT(field)` | Same aggregate | +| `COUNT(DISTINCT field)` | `COUNTD(field)` | +| `+`, `-`, `*`, `/`, parentheses, numeric constants | Explicitly grouped arithmetic | +| Searched `CASE WHEN` | `IF … THEN … ELSEIF … ELSE … END` | +| Comparisons, `AND`, `OR`, `NOT` | Equivalent grouped operators | +| `COALESCE(a, b, …)` | Nested `IFNULL` | +| `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | +| `IS NULL`, `IS NOT NULL` | `ISNULL`, `NOT ISNULL` | +| `ABS`, `ROUND`, `CEIL`, `FLOOR` | `ABS`, `ROUND`, `CEILING`, `FLOOR` | + +These constructs compose. For example, with declared numeric fields `profit` and +`revenue` in dataset `orders`: + +```yaml +metrics: + - name: margin + datatype: Decimal + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0) +``` + +The resulting expression is: + +```text +(SUM([orders].[profit]) / (IF (SUM([orders].[revenue]) = 0) THEN NULL ELSE SUM([orders].[revenue]) END)) +``` + +**Binding and types.** References resolve against declared dataset and field +names, then against fields actually emitted to Salesforce. Physical column names +and source paths are not aliases. Unqualified SQL fields must be unique. Regular +SQL names normalize to uppercase; double-quoted names match the normalized +declaration exactly, following the [expression specification](../../core-spec/expression_language.md). +Thus `"ORDERS"."AMOUNT"` matches regular declarations `orders.amount`, while +`"orders"."amount"` requires explicitly quoted lowercase declarations. Target +API names are preserved; conversion does not rename fields or discover columns. +Names containing brackets or control characters fail because their Tua escaping +is not established. + +`TABLEAU` references use exact `[dataset].[field]` API names. Its supported formula +subset is the Tua equivalents above, including `IF`, `IFNULL`, `ISNULL`, `COUNTD` +and `CEILING`. Existing `ANSI_SQL` expressions using complete bracket notation +retain that spelling as a compatibility case and receive the same validation. +Bracket notation is not accepted as Snowflake SQL. + +Fields need a known compatible datatype, either declared in OSI or restored from +an existing Salesforce field type. Arithmetic and `SUM`/`AVG` require numbers; +`MIN`/`MAX` also permit text and temporal values inside numeric calculations. +Comparisons and conditional/null-handling branches must have compatible types. +Metrics must return numbers; a declared `Integer` result cannot conceal a +fractional expression. Missing metric types are inferred. A formula must be +aggregated or constant: mixed row/aggregate expressions, nested aggregates, and +aggregates without a dataset field fail. A single aggregate cannot combine fields +from multiple datasets. Separate aggregates can use datasets connected through +exported relationships; connectivity alone does not prove join grain. + +**Limits and compatibility.** `COUNT`/`COUNTD` take a field. `ROUND` supports one +argument or a second integer-literal precision; rounding-mode overloads are not +supported. `CEIL`/`FLOOR` take one argument. Simple `CASE`, date/time and string +functions, casts, metric/calculated-field references, windows, LOD, `COUNT(*)`, +SQL comments and backslash string escapes are outside this subset. String and +Boolean literals are supported in predicates. No null-to-zero setting or implicit +cast is added. Errors name the metric and explain the rejected construct or +reference. Literal zero divisors fail; use `NULLIF` to make a zero denominator +nullable. Expressions have bounded size, nesting and generated output. + +This replaces the unvalidated SQL fallback introduced in +[#402](https://github.com/apache/ossie/pull/402). Previously accepted invalid, +unsupported or untyped formulas now fail, including invalid `TABLEAU` input. +Metric/model extension preservation remains owned by #402. CLI conversion errors +are printed to stderr with exit code 3; this small change overlaps the conversion +error handling in [#286](https://github.com/apache/ossie/pull/286). + +**Implementation choice.** The converter reuses its mapping pipeline, datatype +mapper and exceptions. Its bounded recursive-descent parser synthesizes a typed +Tua formula and aggregation level at each production. Add expression support in +the relevant parser/function rule with type, aggregation and composition tests. +It adds no dependencies. The open [#222](https://github.com/apache/ossie/pull/222) +implements an Ossie SQLGlot dialect in Python; it has no Tua emitter or model-bound +field/type checks. Using it here would require a Python runtime bridge and the +same target checks. The Java implementation follows the specification's naming +rules and SQL `NOT` precedence, including #222's proposed precedence correction, +without implementing a new shared expression framework. + +**Validation boundary.** Unit tests cover parsing, binding and failure behavior. +Independent local Tua evaluation tests use synthetic rows with nulls, duplicates, +empty inputs and zero denominators. Those tests model the intended semantics; +they are not native Tableau Next execution. The published Salesforce **output** +schema checks structure, not formula syntax, catalog bindings or authoring API +acceptance. Before deployment, validate authoring and native queries in a test +org, including numeric precision, rounding ties, empty groups, null behavior and +unguarded dynamic division and multi-dataset grain. This converter does not provision Data 360 bindings or enrich +missing fields. + ## Architecture ``` diff --git a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java index ee44b27f..c6e59090 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java @@ -52,6 +52,7 @@ public static void main(String[] args) { } catch (InvalidInputException e) { System.exit(2); } catch (ConversionException e) { + System.err.println("Error: " + e.getMessage()); System.exit(3); } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java new file mode 100644 index 00000000..14caf724 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.*; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.ossie.exception.ConversionException; + +/** + * Compiles the supported metric-expression subset to Salesforce's Tua grammar. + * + *

This is deliberately an expression parser, not a SQL statement parser. Each + * production returns a typed formula with its aggregation level; unsupported + * syntax cannot fall through as untranslated text. See the README for the + * relationship to the proposed Python ossie_sql engine and extension points. + */ +final class MetricExpressionTranslator { + private static final List DIALECTS = List.of("TABLEAU", "SNOWFLAKE", "ANSI_SQL"); + private static final Set AGGREGATES = Set.of("SUM", "AVG", "MIN", "MAX", "COUNT", "COUNTD"); + + record Result(String expression, String dataType) {} + + private MetricExpressionTranslator() {} + + static Result translate(Map metric, Map sourceModel, + Map targetModel) { + String name = getString(metric, "name"); + try { + Map expression = getMap(metric, "expression"); + List dialects = expression == null ? null : getList(expression, "dialects"); + if (dialects == null) { + throw new IllegalArgumentException("missing expression.dialects; provide TABLEAU, SNOWFLAKE or ANSI_SQL"); + } + Map candidates = new java.util.LinkedHashMap<>(); + for (Object entry : dialects) { + Map value = asMap(entry); + String dialect = getString(value, "dialect"); + if (DIALECTS.contains(dialect)) { + if (candidates.containsKey(dialect)) { + throw new IllegalArgumentException("ambiguous expression: multiple " + dialect + " entries"); + } + candidates.put(dialect, getString(value, "expression")); + } + } + String dialect = DIALECTS.stream().filter(candidates::containsKey).findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "no supported dialect; provide TABLEAU, SNOWFLAKE or ANSI_SQL")); + String text = candidates.get(dialect); + if (text == null || text.isBlank()) { + throw new IllegalArgumentException(dialect + " expression is empty"); + } + MetricFieldResolver resolver = new MetricFieldResolver(sourceModel, targetModel); + Parser parser = new Parser(text, dialect, resolver); + Value result = parser.parse(); + resolver.validateDatasets(result.datasets); + Type declared = type(getString(metric, "datatype")); + if (metric.containsKey("datatype") && declared == Type.UNKNOWN) { + throw new IllegalArgumentException("unsupported metric datatype " + getString(metric, "datatype")); + } + if (!result.type.numeric() && result.type != Type.NULL) { + throw new IllegalArgumentException("calculated measurements must be numeric, found " + result.type); + } + if (declared != Type.UNKNOWN && (!declared.numeric() + || (declared == Type.INTEGER && result.type != Type.INTEGER && result.type != Type.NULL))) { + throw new IllegalArgumentException("datatype " + getString(metric, "datatype") + + " is incompatible with expression result " + result.type); + } + if (result.type == Type.NULL && !declared.numeric()) { + throw new IllegalArgumentException("all-null result needs an explicit numeric datatype"); + } + if (result.level == Level.ROW) { + throw new IllegalArgumentException("unaggregated field in metric; use an explicit aggregate"); + } + return new Result(result.formula, "Number"); + } catch (IllegalArgumentException e) { + throw new ConversionException("Metric '" + name + "': " + e.getMessage(), e); + } + } + + private enum Level { CONSTANT, ROW, AGGREGATE } + + private enum Type { + INTEGER, DECIMAL, FLOAT, STRING, BOOLEAN, DATE, DATETIME, DATETIME_TZ, NULL, UNKNOWN; + boolean numeric() { return this == INTEGER || this == DECIMAL || this == FLOAT; } + } + + private static Type type(String datatype) { + if (datatype == null) return Type.UNKNOWN; + return switch (datatype) { + case "Integer" -> Type.INTEGER; + case "Decimal" -> Type.DECIMAL; + case "Float" -> Type.FLOAT; + case "String" -> Type.STRING; + case "Boolean" -> Type.BOOLEAN; + case "Date" -> Type.DATE; + case "DateTime" -> Type.DATETIME; + case "DateTimeTz" -> Type.DATETIME_TZ; + default -> Type.UNKNOWN; + }; + } + + /** A synthesized parser attribute, not a second general-purpose expression model. */ + private record Value(String formula, Type type, Level level, Set datasets, + boolean field, BigDecimal number) { + Value(String formula, Type type, Level level, Set datasets) { + this(formula, type, level, datasets, false, null); + } + } + + private enum Kind { WORD, IDENTIFIER, STRING, NUMBER, SYMBOL, END } + private record Token(Kind kind, String text, int offset, boolean bracket) {} + + private static final class Parser { + private final List tokens; + private final String dialect; + private final MetricFieldResolver resolver; + private int position; + private int depth; + + Parser(String expression, String dialect, MetricFieldResolver resolver) { + this.dialect = dialect; + this.resolver = resolver; + this.tokens = tokenize(expression, dialect); + } + + Value parse() { + Value value = expression(); + if (peek().kind != Kind.END) throw error("unsupported or unexpected token '" + peek().text + "'"); + return value; + } + + private Value expression() { + if (++depth > 128) throw error("expression nesting exceeds 128 levels"); + Value value = or(); + depth--; + return value; + } + + private Value or() { + Value value = and(); + while (take("OR")) value = binary("OR", value, and(), Type.BOOLEAN); + return value; + } + + private Value and() { + Value value = not(); + while (take("AND")) value = binary("AND", value, not(), Type.BOOLEAN); + return value; + } + + // SQL NOT binds below comparisons, unlike the old draft's precedence table. + private Value not() { + int count = 0; + while (take("NOT")) { + if (++count > 128) throw error("too many unary operators"); + } + Value value = comparison(); + for (int i = 0; i < count; i++) { + require(value, Type.BOOLEAN, "NOT"); + value = new Value("(NOT " + value.formula + ")", Type.BOOLEAN, value.level, value.datasets); + } + return value; + } + + private Value comparison() { + Value value = additive(); + if (take("IS")) { + if (dialect.equals("TABLEAU")) throw error("use ISNULL in TABLEAU expressions"); + boolean negated = take("NOT"); + expect("NULL"); + return new Value((negated ? "(NOT ISNULL(" : "ISNULL(") + value.formula + + (negated ? "))" : ")"), Type.BOOLEAN, value.level, value.datasets); + } + if (Set.of("=", "!=", "<>", "<", "<=", ">", ">=").contains(peek().text)) { + String operator = next().text; + Value right = additive(); + compatible(value.type, right.type, "comparison"); + if (!Set.of("=", "!=", "<>").contains(operator) + && (value.type == Type.BOOLEAN || right.type == Type.BOOLEAN)) { + throw error("ordered comparison requires numeric, text or temporal operands"); + } + return compose("(" + value.formula + " " + (operator.equals("<>") ? "!=" : operator) + + " " + right.formula + ")", Type.BOOLEAN, List.of(value, right)); + } + return value; + } + + private Value additive() { + Value value = multiplicative(); + while (at("+") || at("-")) { + String operator = next().text; + Value right = multiplicative(); + value = binary(operator, value, right, numericType(value, right, operator)); + } + return value; + } + + private Value multiplicative() { + Value value = unary(); + while (at("*") || at("/")) { + String operator = next().text; + Value right = unary(); + if (operator.equals("/") && right.number != null && right.number.signum() == 0) { + throw error("division by literal zero; use NULLIF(denominator, 0) for a nullable denominator"); + } + Type resultType = numericType(value, right, operator); + value = binary(operator, value, right, operator.equals("/") ? Type.DECIMAL : resultType); + } + return value; + } + + private Value unary() { + List signs = new ArrayList<>(); + while (at("+") || at("-")) { + if (signs.size() >= 128) throw error("too many unary operators"); + signs.add(next().text); + } + Value value = primary(); + for (int i = signs.size() - 1; i >= 0; i--) { + numeric(value, "unary " + signs.get(i)); + boolean negative = signs.get(i).equals("-"); + value = new Value(negative ? "(-" + value.formula + ")" : value.formula, + value.type, value.level, value.datasets, false, + value.number == null ? null : negative ? value.number.negate() : value.number); + } + return value; + } + + private Value primary() { + if (take("(")) { + Value value = expression(); + expect(")"); + return value; + } + if (at("CASE")) { + if (dialect.equals("TABLEAU")) throw error("searched CASE is SQL; use IF in TABLEAU"); + next(); + return conditional(false); + } + if (at("IF") && dialect.equals("TABLEAU")) { + next(); + return conditional(true); + } + if (take("NULL")) return new Value("NULL", Type.NULL, Level.CONSTANT, Set.of()); + if (at("TRUE") || at("FALSE")) { + return new Value(next().text.toUpperCase(Locale.ROOT), Type.BOOLEAN, Level.CONSTANT, Set.of()); + } + Token token = next(); + if (token.kind == Kind.NUMBER) { + BigDecimal number; + try { number = new BigDecimal(token.text); } + catch (NumberFormatException e) { throw error("invalid numeric literal '" + token.text + "'"); } + if (Math.abs((long) number.scale()) > 1000 || number.precision() > 1000) { + throw error("numeric literal is too large"); + } + Type datatype = number.stripTrailingZeros().scale() <= 0 ? Type.INTEGER : Type.DECIMAL; + return new Value(number.toPlainString(), datatype, Level.CONSTANT, Set.of(), false, number); + } + if (token.kind == Kind.STRING) { + return new Value("'" + token.text.replace("'", "''") + "'", Type.STRING, Level.CONSTANT, Set.of()); + } + if (token.kind != Kind.WORD && token.kind != Kind.IDENTIFIER) { + throw error("expected a value, found '" + token.text + "'"); + } + if (take("(")) { + if (token.kind != Kind.WORD) throw error("quoted function names are unsupported"); + return function(token.text.toUpperCase(Locale.ROOT)); + } + List parts = new ArrayList<>(); + addIdentifier(parts, token); + boolean bracketed = token.bracket; + while (take(".")) { + Token part = next(); + if (bracketed != part.bracket) throw error("do not mix bracketed and SQL field identifiers"); + addIdentifier(parts, part); + } + // Existing ANSI_SQL fixtures use complete Tableau field notation. Keep + // that narrow compatibility spelling, with the same exact-name checks. + MetricFieldResolver.ResolvedField field = resolver.resolve(parts, bracketed); + return new Value(field.expression(), type(field.datatype()), Level.ROW, + Set.of(field.dataset()), true, null); + } + + private void addIdentifier(List parts, Token token) { + if (token.kind != Kind.WORD && token.kind != Kind.IDENTIFIER) throw error("expected field identifier"); + if (dialect.equals("TABLEAU") && !token.bracket) { + throw error("TABLEAU fields must use [dataset].[field] notation"); + } + parts.add(new MetricFieldResolver.Identifier(token.text, token.kind == Kind.IDENTIFIER)); + } + + private Value conditional(boolean tableau) { + List values = new ArrayList<>(); + StringBuilder formula = new StringBuilder("(IF "); + Type resultType = Type.NULL; + boolean first = true; + do { + if (!first) formula.append(" ELSEIF "); + if (!tableau) expect("WHEN"); + Value condition = expression(); + require(condition, Type.BOOLEAN, "conditional predicate"); + expect("THEN"); + Value branch = expression(); + resultType = compatible(resultType, branch.type, "conditional branches"); + values.add(condition); + values.add(branch); + formula.append(condition.formula).append(" THEN ").append(branch.formula); + first = false; + } while (tableau ? take("ELSEIF") : at("WHEN")); + Value otherwise = take("ELSE") ? expression() : new Value("NULL", Type.NULL, Level.CONSTANT, Set.of()); + expect("END"); + resultType = compatible(resultType, otherwise.type, "conditional branches"); + values.add(otherwise); + formula.append(" ELSE ").append(otherwise.formula).append(" END)"); + return compose(formula.toString(), resultType, values); + } + + private Value function(String name) { + if (++depth > 128) throw error("expression nesting exceeds 128 levels"); + boolean distinct = take("DISTINCT"); + if (at("*")) throw error("COUNT(*) is unsupported; name a declared field to count"); + List arguments = new ArrayList<>(); + if (!at(")")) { + do { arguments.add(expression()); } while (take(",")); + } + expect(")"); + depth--; + if (distinct && (!name.equals("COUNT") || dialect.equals("TABLEAU"))) { + throw error("DISTINCT is supported only by SQL COUNT(DISTINCT field)"); + } + if (AGGREGATES.contains(name)) { + if (name.equals("COUNTD") && !dialect.equals("TABLEAU")) throw error("use COUNT(DISTINCT field) in SQL"); + arity(name, arguments, 1, 1); + Value argument = arguments.get(0); + if (argument.level == Level.AGGREGATE) throw error("nested aggregate " + name + " is unsupported"); + if (argument.datasets.isEmpty()) throw error(name + " needs a declared field to establish its dataset"); + if (argument.datasets.size() > 1) throw error("one aggregate cannot combine fields from multiple datasets"); + boolean count = name.equals("COUNT") || name.equals("COUNTD"); + if (count && !argument.field) throw error(name + " requires a declared field; counting expressions is unsupported"); + if (name.equals("MIN") || name.equals("MAX")) { + if (argument.type == Type.BOOLEAN || argument.type == Type.UNKNOWN) { + throw error(name + " requires numeric, text or temporal operands"); + } + } else if (!count) numeric(argument, name); + return new Value((distinct ? "COUNTD" : name) + "(" + argument.formula + ")", + count ? Type.INTEGER : name.equals("AVG") ? Type.DECIMAL : argument.type, + Level.AGGREGATE, argument.datasets); + } + if (Set.of("COALESCE", "NULLIF").contains(name) && dialect.equals("TABLEAU")) { + throw error(name + " is SQL syntax; use IFNULL or IF in TABLEAU"); + } + if (Set.of("IFNULL", "ISNULL", "CEILING").contains(name) && !dialect.equals("TABLEAU")) { + throw error(name + " is outside the supported SQL subset"); + } + return switch (name) { + case "COALESCE", "IFNULL" -> coalesce(name, arguments); + case "NULLIF" -> nullif(arguments); + case "ISNULL" -> { + arity(name, arguments, 1, 1); + yield compose(call(name, arguments), Type.BOOLEAN, arguments); + } + case "ABS", "CEIL", "CEILING", "FLOOR", "ROUND" -> numericFunction(name, arguments); + default -> throw error("unsupported function " + name); + }; + } + + private Value coalesce(String name, List arguments) { + arity(name, arguments, 2, name.equals("IFNULL") ? 2 : Integer.MAX_VALUE); + Type resultType = Type.NULL; + for (Value argument : arguments) resultType = compatible(resultType, argument.type, name + " arguments"); + String formula = arguments.get(arguments.size() - 1).formula; + for (int i = arguments.size() - 2; i >= 0; i--) formula = "IFNULL(" + arguments.get(i).formula + ", " + formula + ")"; + return compose(formula, resultType, arguments); + } + + private Value nullif(List arguments) { + arity("NULLIF", arguments, 2, 2); + Value left = arguments.get(0); + Value right = arguments.get(1); + compatible(left.type, right.type, "NULLIF arguments"); + return compose("(IF (" + left.formula + " = " + right.formula + ") THEN NULL ELSE " + + left.formula + " END)", left.type, arguments); + } + + private Value numericFunction(String name, List arguments) { + if (name.equals("CEIL") && dialect.equals("TABLEAU")) throw error("use CEILING in TABLEAU"); + arity(name, arguments, 1, name.equals("ROUND") ? 2 : 1); + Value value = arguments.get(0); + numeric(value, name); + if (arguments.size() == 2) { + BigDecimal places = arguments.get(1).number; + if (places == null || places.stripTrailingZeros().scale() > 0 + || places.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0 + || places.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { + throw error("ROUND precision must be a 32-bit integer literal"); + } + } + String target = name.equals("CEIL") ? "CEILING" : name; + return compose(call(target, arguments), value.type, arguments); + } + + private String call(String name, List arguments) { + return name + "(" + String.join(", ", arguments.stream().map(Value::formula).toList()) + ")"; + } + + private void arity(String name, List arguments, int min, int max) { + if (arguments.size() < min || arguments.size() > max) { + throw error(name + " expects " + (min == max ? min : min + " to " + max) + " arguments"); + } + } + + private Value binary(String operator, Value left, Value right, Type resultType) { + if (operator.equals("AND") || operator.equals("OR")) { + require(left, Type.BOOLEAN, operator); + require(right, Type.BOOLEAN, operator); + } + return compose("(" + left.formula + " " + operator + " " + right.formula + ")", + resultType, List.of(left, right)); + } + + private Type numericType(Value left, Value right, String context) { + numeric(left, context); + numeric(right, context); + return compatible(left.type, right.type, context); + } + + private void numeric(Value value, String context) { + if (!value.type.numeric() && value.type != Type.NULL) { + throw error(context + " requires numeric operands, found " + value.type + + "; declare a compatible field datatype"); + } + } + + private void require(Value value, Type expected, String context) { + if (value.type != expected && value.type != Type.NULL) throw error(context + " requires " + expected + ", found " + value.type); + } + + private Type compatible(Type left, Type right, String context) { + if (left == Type.UNKNOWN || right == Type.UNKNOWN) throw error(context + " needs known field datatypes"); + if (left == Type.NULL) return right; + if (right == Type.NULL || left == right) return left; + if (left.numeric() && right.numeric()) { + if (left == Type.FLOAT || right == Type.FLOAT) return Type.FLOAT; + return Type.DECIMAL; + } + throw error(context + " has incompatible types " + left + " and " + right); + } + + private Value compose(String formula, Type type, List arguments) { + Level level = Level.CONSTANT; + Set datasets = new HashSet<>(); + for (Value argument : arguments) { + if (level != Level.CONSTANT && argument.level != Level.CONSTANT && level != argument.level) { + throw error("cannot mix aggregate and unaggregated field expressions"); + } + if (argument.level != Level.CONSTANT) level = argument.level; + datasets.addAll(argument.datasets); + } + // NULLIF duplicates its first operand. Bound expansion as well as input size. + if (formula.length() > 131072) throw error("translated expression exceeds 131072 characters"); + return new Value(formula, type, level, Set.copyOf(datasets)); + } + + private Token peek() { return tokens.get(position); } + private Token next() { Token token = peek(); if (token.kind != Kind.END) position++; return token; } + private boolean at(String text) { + return (peek().kind == Kind.WORD || peek().kind == Kind.SYMBOL) && peek().text.equalsIgnoreCase(text); + } + private boolean take(String text) { if (!at(text)) return false; next(); return true; } + private void expect(String text) { if (!take(text)) throw error("expected " + text + ", found '" + peek().text + "'"); } + private IllegalArgumentException error(String message) { + return new IllegalArgumentException(dialect + " at character " + (peek().offset + 1) + ": " + message); + } + } + + private static List tokenize(String text, String dialect) { + if (text.length() > 32768) throw new IllegalArgumentException("expression exceeds 32768 characters"); + List tokens = new ArrayList<>(); + for (int i = 0; i < text.length();) { + char c = text.charAt(i); + if (Character.isWhitespace(c)) { i++; continue; } + int start = i; + if (c == '\'' || c == '"' || c == '[') { + if (c == '[' && dialect.equals("SNOWFLAKE")) throw lexical(dialect, i, "use double-quoted SQL identifiers"); + boolean string = c == '\'' || (c == '"' && dialect.equals("TABLEAU")); + char end = c == '[' ? ']' : c; + StringBuilder value = new StringBuilder(); + boolean closed = false; + i++; + while (i < text.length()) { + char part = text.charAt(i++); + if (part == end) { + if (i < text.length() && text.charAt(i) == end) { value.append(end); i++; } + else { closed = true; break; } + } else { + if (Character.isISOControl(part) || (string && part == '\\')) { + throw lexical(dialect, i - 1, "control characters and backslash string escapes are unsupported"); + } + value.append(part); + } + } + if (!closed) throw lexical(dialect, start, "unterminated quoted value"); + tokens.add(new Token(string ? Kind.STRING : Kind.IDENTIFIER, value.toString(), start, c == '[')); + } else if (Character.isDigit(c) || (c == '.' && i + 1 < text.length() && Character.isDigit(text.charAt(i + 1)))) { + i++; + while (i < text.length() && (Character.isDigit(text.charAt(i)) || text.charAt(i) == '.')) i++; + if (i < text.length() && (text.charAt(i) == 'e' || text.charAt(i) == 'E')) { + i++; + if (i < text.length() && (text.charAt(i) == '+' || text.charAt(i) == '-')) i++; + while (i < text.length() && Character.isDigit(text.charAt(i))) i++; + } + tokens.add(new Token(Kind.NUMBER, text.substring(start, i), start, false)); + } else if (Character.isLetter(c) || c == '_') { + i++; + while (i < text.length() && (Character.isLetterOrDigit(text.charAt(i)) || text.charAt(i) == '_' || text.charAt(i) == '$')) i++; + tokens.add(new Token(Kind.WORD, text.substring(start, i), start, false)); + } else { + if (i + 1 < text.length() && (text.startsWith("--", i) || text.startsWith("/*", i))) { + throw lexical(dialect, i, "comments are unsupported in metric expressions"); + } + String symbol = String.valueOf(c); + if (i + 1 < text.length() && Set.of("<=", ">=", "<>", "!=").contains(text.substring(i, i + 2))) { + symbol = text.substring(i, i + 2); + i++; + } + if (!"()+-*/.,=<>!".contains(String.valueOf(c))) throw lexical(dialect, start, "unsupported character '" + c + "'"); + tokens.add(new Token(Kind.SYMBOL, symbol, start, false)); + i++; + } + if (tokens.size() > 8192) throw lexical(dialect, start, "too many expression tokens"); + } + tokens.add(new Token(Kind.END, "end of expression", text.length(), false)); + return tokens; + } + + private static IllegalArgumentException lexical(String dialect, int offset, String message) { + return new IllegalArgumentException(dialect + " at character " + (offset + 1) + ": " + message); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java new file mode 100644 index 00000000..5da5dcc8 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.getList; +import static org.apache.ossie.util.DataStructureUtils.getString; +import static org.apache.ossie.util.DataStructureUtils.streamMaps; + +import java.util.ArrayList; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** Binds metric references to declared fields that the Salesforce converter actually exported. */ +final class MetricFieldResolver { + + record Identifier(String text, boolean quoted) {} + + record ResolvedField(String expression, String datatype, String dataset) {} + + private record Field(Map dataset, Map field) {} + + private final List> datasets; + private final List> targetDatasets; + private final List> relationships; + + MetricFieldResolver(Map sourceModel, Map targetModel) { + datasets = items(sourceModel, "datasets"); + targetDatasets = items(targetModel, "semanticDataObjects"); + relationships = items(targetModel, "semanticRelationships"); + } + + /** Checks exported connectivity only; join grain and cardinality still require native validation. */ + void validateDatasets(Set referenced) { + if (referenced.size() < 2) { + return; + } + Map> graph = new HashMap<>(); + for (Map dataset : targetDatasets) { + String name = getString(dataset, "apiName"); + if (name != null) { + graph.put(name, new HashSet<>()); + } + } + for (Map relationship : relationships) { + String left = getString(relationship, "leftSemanticDefinitionApiName"); + String right = getString(relationship, "rightSemanticDefinitionApiName"); + if (graph.containsKey(left) && graph.containsKey(right)) { + graph.get(left).add(right); + graph.get(right).add(left); + } + } + Set visited = new HashSet<>(); + ArrayDeque pending = new ArrayDeque<>(); + pending.add(referenced.iterator().next()); + while (!pending.isEmpty()) { + String dataset = pending.removeFirst(); + if (visited.add(dataset)) { + pending.addAll(graph.getOrDefault(dataset, Set.of())); + } + } + if (!visited.containsAll(referenced)) { + throw new IllegalArgumentException("Metric references disconnected datasets " + + referenced.stream().sorted().collect(Collectors.joining(", ")) + + "; declare supported relationships connecting them before exporting the metric"); + } + } + + ResolvedField resolve(List parts, boolean tableau) { + String reference = parts.stream().map(Identifier::text).collect(Collectors.joining(".")); + if (parts.isEmpty() || parts.size() > 2) { + throw new IllegalArgumentException("Reference '" + reference + + "' must name a declared field or dataset.field; physical source paths are unsupported"); + } + if (tableau && parts.size() != 2) { + throw new IllegalArgumentException("TABLEAU field reference '" + reference + + "' must use [dataset].[field]"); + } + + List> candidates = datasets; + if (parts.size() == 2) { + candidates = datasets.stream() + .filter(dataset -> matches(parts.get(0), getString(dataset, "name"), tableau)) + .toList(); + if (candidates.isEmpty()) { + throw new IllegalArgumentException("Unknown dataset in reference '" + reference + + "'; use a declared dataset name, not its physical source"); + } + if (candidates.size() > 1) { + throw new IllegalArgumentException("Ambiguous dataset in reference '" + reference + + "'; dataset declarations must have distinct names"); + } + } + + Identifier fieldName = parts.get(parts.size() - 1); + List fields = new ArrayList<>(); + for (Map dataset : candidates) { + for (Map field : items(dataset, "fields")) { + if (matches(fieldName, getString(field, "name"), tableau)) { + fields.add(new Field(dataset, field)); + } + } + } + if (fields.isEmpty()) { + throw new IllegalArgumentException("Unknown field reference '" + reference + + "'; declare the field under datasets[].fields before exporting the metric"); + } + if (fields.size() > 1) { + throw new IllegalArgumentException("Ambiguous field reference '" + reference + + "'; qualify the dataset and remove duplicate field declarations"); + } + + Field match = fields.get(0); + String datasetName = getString(match.dataset(), "name"); + // An unqualified field must not accidentally select one of two equivalent datasets. + if (datasets.stream().filter(dataset -> equivalentDeclaration( + datasetName, getString(dataset, "name"), tableau)).count() > 1) { + throw new IllegalArgumentException("Ambiguous dataset for reference '" + reference + + "'; dataset declarations must have distinct names"); + } + String sourceFieldName = getString(match.field(), "name"); + Map targetDataset = exportedItem(targetDatasets, datasetName, + "dataset", reference); + List> targetFields = new ArrayList<>(items(targetDataset, "semanticDimensions")); + targetFields.addAll(items(targetDataset, "semanticMeasurements")); + Map targetField = exportedItem(targetFields, sourceFieldName, "field", reference); + + String datatype = getString(match.field(), "datatype"); + String targetType = getString(targetField, "dataType"); + if (datatype == null || datatype.isBlank()) { + datatype = SalesforceDataTypeMapper.toOssie(targetType); + } + if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { + throw new IllegalArgumentException("Field reference '" + reference + + "' has no supported datatype; declare a portable field datatype"); + } + if (targetType == null || targetType.isBlank() + || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { + throw new IllegalArgumentException("Field reference '" + reference + "' has datatype '" + + datatype + "' but exported Salesforce dataType '" + targetType + + "'; use compatible field types"); + } + + String targetDatasetName = getString(targetDataset, "apiName"); + String targetFieldName = getString(targetField, "apiName"); + return new ResolvedField(bracket(targetDatasetName) + "." + bracket(targetFieldName), + datatype, targetDatasetName); + } + + private static Map exportedItem(List> items, + String name, String kind, String reference) { + List> matches = items.stream() + .filter(item -> name.equals(getString(item, "apiName"))).toList(); + if (matches.isEmpty()) { + throw new IllegalArgumentException("Declared " + kind + " in reference '" + reference + + "' was not exported as a direct Salesforce semantic " + kind + + "; calculated or omitted fields are unsupported in metric references"); + } + if (matches.size() > 1) { + throw new IllegalArgumentException("Ambiguous exported Salesforce " + kind + " for reference '" + + reference + "'; apiName values must be unique"); + } + return matches.get(0); + } + + private static boolean matches(Identifier reference, String declaration, boolean tableau) { + if (declaration == null) { + return false; + } + return tableau ? reference.text().equals(declaration) + : normalize(reference).equals(normalizeDeclaration(declaration)); + } + + private static boolean equivalentDeclaration(String first, String second, boolean tableau) { + return second != null && (tableau ? first.equals(second) + : normalizeDeclaration(first).equals(normalizeDeclaration(second))); + } + + private static String normalize(Identifier identifier) { + return identifier.quoted() ? identifier.text() : identifier.text().toUpperCase(Locale.ROOT); + } + + private static String normalizeDeclaration(String name) { + if (name.startsWith("\"") && name.endsWith("\"") && name.length() >= 2) { + String text = name.substring(1, name.length() - 1); + String unescaped = text.replace("\"\"", ""); + if (text.isEmpty() || unescaped.contains("\"")) { + throw new IllegalArgumentException("Invalid quoted declaration name '" + name + "'"); + } + return text.replace("\"\"", "\""); + } + return name.toUpperCase(Locale.ROOT); + } + + private static String bracket(String name) { + if (name == null || name.isBlank() || name.indexOf('[') >= 0 || name.indexOf(']') >= 0 + || name.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("Exported apiName '" + name + + "' cannot be represented safely in a TABLEAU field reference; rename it"); + } + return "[" + name + "]"; + } + + private static List> items(Map map, String key) { + List values = getList(map, key); + return values == null ? List.of() : streamMaps(values).toList(); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 00de289d..c258fc34 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -77,6 +77,14 @@ private void mapOssieToSalesforce( return; } + Set names = new HashSet<>(); + for (Object metric : ossieMetrics) { + String name = getString(asMap(metric), NAME); + if (!names.add(name)) { + throw new ConversionException("Metric '" + name + "': duplicate metric name"); + } + } + // Filter mappings to get only metric-related entries Map metricMappings = MappingUtils.filterMappingsByPrefix(mappings, METRICS); @@ -87,7 +95,10 @@ private void mapOssieToSalesforce( List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); if (sfMetrics != null) { - unwrapExpressions(ossieMetrics, sfMetrics); + unwrapExpressions(ossieMetrics, sfMetrics, sourceData, outputData); + } else if (!ossieMetrics.isEmpty()) { + throw new ConversionException("Metric '" + getString(asMap(ossieMetrics.get(0)), NAME) + + "': metric mappings produced no calculated measurements"); } } @@ -127,64 +138,25 @@ private void mapSalesforceToOssie( /** - * Unwraps expressions for Ossie→SF conversion, mirroring {@link #wrapExpressions}. - * - *

Picks an expression out of each Ossie metric's {@code expression.dialects[]} and - * flattens it into the Salesforce metric's {@code expression} string. {@code TABLEAU} is - * preferred (it is what Salesforce/Tableau CRM itself speaks); a model authored without one - * falls back to {@code ANSI_SQL} best-effort, since resolving/rewriting an expression into - * TABLEAU syntax is the scope of #222's expression-language work, not this fix. A metric with - * neither dialect fails the conversion rather than being silently omitted (#399). + * Compiles each metric to Tua after fields have been mapped. Binding checks both + * the OSI declarations and the actual emitted fields, including their types. */ - private void unwrapExpressions(List ossieMetrics, List sfMetrics) { - for (int i = 0; i < ossieMetrics.size() && i < sfMetrics.size(); i++) { + private void unwrapExpressions(List ossieMetrics, List sfMetrics, + Map sourceData, Map outputData) { + if (ossieMetrics.size() != sfMetrics.size()) { + throw new ConversionException("Metric export count differs from declared metrics: " + + streamMaps(ossieMetrics).map(metric -> getString(metric, NAME)).toList()); + } + for (int i = 0; i < ossieMetrics.size(); i++) { Map ossieMetric = asMap(ossieMetrics.get(i)); Map sfMetric = asMap(sfMetrics.get(i)); - - String expressionValue = extractExpression(ossieMetric, DIALECT_TABLEAU); - if (expressionValue == null) { - expressionValue = extractExpression(ossieMetric, DIALECT_ANSI_SQL); - if (expressionValue != null) { - logger.warn( - "Metric '{}' has no TABLEAU-dialect expression; exporting its " - + "ANSI_SQL expression to Salesforce unresolved/untranslated", - getString(ossieMetric, NAME)); - } - } - if (expressionValue == null) { - throw new ConversionException( - "Metric '" + getString(ossieMetric, NAME) + "' has neither a TABLEAU nor " - + "an ANSI_SQL expression to export to Salesforce; add one to " - + "expression.dialects[] or remove the metric."); - } - sfMetric.put(EXPRESSION, expressionValue); - - String datatype = SalesforceDataTypeMapper.toSalesforce(getString(ossieMetric, OSSIE_DATATYPE)); - if (datatype != null) { - sfMetric.put(DATA_TYPE, datatype); - } - } - } - - /** - * Finds the given dialect's expression string in an Ossie metric's - * {@code expression.dialects[]}, or {@code null} when the metric has no expression or no - * entry for that dialect. - */ - private String extractExpression(Map ossieMetric, String dialect) { - Map expression = getMap(ossieMetric, EXPRESSION); - if (expression == null) { - return null; - } - List dialects = getList(expression, DIALECTS); - if (dialects == null) { - return null; + MetricExpressionTranslator.Result translated = + MetricExpressionTranslator.translate(ossieMetric, sourceData, outputData); + sfMetric.put(EXPRESSION, translated.expression()); + sfMetric.put(DATA_TYPE, translated.dataType()); + sfMetric.put("syntax", "Tua"); + sfMetric.put("aggregationType", "UserAgg"); } - return streamMaps(dialects) - .filter(d -> dialect.equals(getString(d, DIALECT))) - .map(d -> getString(d, EXPRESSION)) - .findFirst() - .orElse(null); } /** diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java new file mode 100644 index 00000000..93528956 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import org.apache.ossie.app.OssieSalesforceConverter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class MetricCliTest { + @TempDir + Path directory; + + @Test + void reportsMetricFailureToStderrWithoutWritingAModel() throws Exception { + Path input = directory.resolve("input.yaml"); + Files.writeString(input, Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")) + .replace("SUM([Orders].[amount])", "SUM([Orders].[missing])")); + Path stderr = directory.resolve("stderr.txt"); + Process process = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), OssieSalesforceConverter.class.getName(), + "toSF", input.toString()) + .redirectError(stderr.toFile()) + .redirectOutput(directory.resolve("stdout.txt").toFile()) + .start(); + try { + assertTrue(process.waitFor(30, TimeUnit.SECONDS), "CLI did not terminate"); + assertEquals(3, process.exitValue()); + String error = Files.readString(stderr); + assertTrue(error.contains("Metric 'total_revenue'"), error); + assertTrue(error.contains("Unknown field reference"), error); + assertTrue(error.contains("missing"), error); + assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); + } finally { + process.destroyForcibly(); + } + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java new file mode 100644 index 00000000..7a001777 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.ossie.converter.ConversionDirection; +import org.apache.ossie.converter.Converter; +import org.apache.ossie.converter.ConverterFactory; +import org.apache.ossie.exception.ConversionException; +import org.apache.ossie.exception.ValidationException; +import org.apache.ossie.validator.SchemaValidator; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** Exercises metric compilation through the public converter and target schema. */ +class MetricExportIntegrationTest { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); + private static boolean salesforceSchemaExists; + private static boolean ossieSchemaExists; + + private Converter converter; + + @TempDir + Path temporaryDirectory; + + @BeforeAll + static void checkSchemaAvailability() { + salesforceSchemaExists = MetricExportIntegrationTest.class + .getResource(SchemaValidator.SALESFORCE_SCHEMA_PATH) != null; + ossieSchemaExists = MetricExportIntegrationTest.class + .getResource(SchemaValidator.OSSIE_SCHEMA_PATH) != null; + if (Boolean.getBoolean("requireSalesforceSchema")) { + assertTrue(salesforceSchemaExists, + "-DrequireSalesforceSchema=true requires the Salesforce schema; see README setup instructions"); + } + } + + @BeforeEach + void setUp() { + assumeTrue(ossieSchemaExists, "Ossie schema is required; see README setup instructions"); + converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void translatesComposedMetricsThroughStringApi(String dialect) throws Exception { + Map output = convertOne(model("sales", List.of( + metric("margin", dialect, "SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0)"), + metric("adjusted_average", dialect, + "ROUND(AVG(CASE WHEN orders.revenue IS NOT NULL AND NOT orders.profit < 0 " + + "THEN COALESCE(orders.profit, 0) ELSE 0 END), 2)"), + metric("customers", dialect, "COUNT(DISTINCT orders.customer_id)")))); + + List> measurements = measurements(output); + assertEquals(List.of("margin", "adjusted_average", "customers"), + measurements.stream().map(item -> item.get("apiName")).toList()); + for (Map measurement : measurements) { + assertMeasurementMetadata(measurement); + } + assertEquals("(SUM([orders].[profit]) / (IF (SUM([orders].[revenue]) = 0) " + + "THEN NULL ELSE SUM([orders].[revenue]) END))", + measurements.get(0).get("expression")); + String conditional = (String) measurements.get(1).get("expression"); + assertTrue(conditional.startsWith("ROUND(AVG((IF "), conditional); + assertTrue(conditional.contains("ISNULL([orders].[revenue])"), conditional); + assertTrue(conditional.contains("IFNULL([orders].[profit], 0)"), conditional); + assertFalse(conditional.contains("CASE"), conditional); + assertEquals("COUNTD([orders].[customer_id])", measurements.get(2).get("expression")); + + Map dataset = items(output, "semanticDataObjects").get(0); + Map profit = items(dataset, "semanticMeasurements").stream() + .filter(item -> item.get("apiName").equals("profit")).findFirst().orElseThrow(); + assertEquals("profit__c", profit.get("dataObjectFieldName")); + assertFalse(measurements.get(0).get("expression").toString().contains("profit__c"), + "A metric binds semantic field names, not physical source columns"); + } + + @Test + void fileApiWritesTheSameCompleteModelAsStringApi() throws Exception { + String input = document(List.of(model("sales", List.of( + metric("revenue", "SNOWFLAKE", "SUM(orders.revenue)"))))); + Path source = temporaryDirectory.resolve("input.yaml"); + Path outputDirectory = Files.createDirectory(temporaryDirectory.resolve("output")); + Files.writeString(source, input); + + converter.convert(source, outputDirectory); + + assertEquals(JSON.readTree(converter.convert(input).get(0)), + JSON.readTree(Files.readString(outputDirectory.resolve("sales.json")))); + try (var files = Files.list(outputDirectory)) { + assertEquals(List.of("sales.json"), files.map(path -> path.getFileName().toString()).toList()); + } + } + + @Test + void missingFieldFailsWithMetricNameAndDoesNotWriteAnyModelFiles() throws Exception { + String input = document(List.of( + model("valid", List.of(metric("revenue", "ANSI_SQL", "SUM(orders.revenue)"))), + model("invalid", List.of(metric("broken_margin", "SNOWFLAKE", "SUM(orders.missing)"))))); + Path source = temporaryDirectory.resolve("input.yaml"); + Path outputDirectory = Files.createDirectory(temporaryDirectory.resolve("output")); + Path existing = outputDirectory.resolve("existing.json"); + Files.writeString(source, input); + Files.writeString(existing, "preserve this file"); + + ConversionException error = assertThrows(ConversionException.class, + () -> converter.convert(source, outputDirectory)); + + assertTrue(error.getMessage().contains("broken_margin"), error.getMessage()); + assertTrue(error.getMessage().contains("orders.missing"), error.getMessage()); + assertTrue(error.getMessage().contains("declare"), error.getMessage()); + assertEquals("preserve this file", Files.readString(existing)); + try (var files = Files.list(outputDirectory)) { + assertEquals(List.of("existing.json"), files.map(path -> path.getFileName().toString()).toList(), + "A failure in a later model must not leave an earlier model's output behind"); + } + } + + @Test + void selectedUnsupportedTableauExpressionDoesNotFallBackToSql() throws Exception { + Map metric = metric("chosen_tableau", "TABLEAU", "BOGUS([orders].[profit])"); + metric.put("expression", Map.of("dialects", List.of( + dialect("ANSI_SQL", "SUM(orders.profit)"), + dialect("TABLEAU", "BOGUS([orders].[profit])"), + dialect("SNOWFLAKE", "SUM(orders.profit)")))); + String input = document(List.of(model("sales", List.of(metric)))); + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("chosen_tableau"), error.getMessage()); + assertTrue(error.getMessage().contains("BOGUS"), error.getMessage()); + } + + @Test + void preservesSupportedTableauExpressionMeaningAndValidatesReferences() throws Exception { + Map output = convertOne(model("sales", List.of( + metric("revenue", "TABLEAU", "IFNULL(SUM([orders].[revenue]), 0)")))); + + Map measurement = measurements(output).get(0); + assertEquals("IFNULL(SUM([orders].[revenue]), 0)", measurement.get("expression")); + assertMeasurementMetadata(measurement); + + String invalid = document(List.of(model("sales", List.of( + metric("unknown_tableau", "TABLEAU", "SUM([orders].[missing])"))))); + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(invalid)); + assertTrue(error.getMessage().contains("unknown_tableau"), error.getMessage()); + assertTrue(error.getMessage().contains("orders.missing"), error.getMessage()); + } + + @Test + void modelWithoutMetricsRetainsDatasetExport() throws Exception { + Map source = model("sales", List.of()); + source.remove("metrics"); + + Map output = convertOne(source); + + assertTrue(measurements(output).isEmpty()); + assertEquals(3, items(items(output, "semanticDataObjects").get(0), "semanticMeasurements").size()); + } + + @Test + void emptyMetricsRemainEmpty() throws Exception { + Map output = convertOne(model("sales", List.of())); + + assertTrue(measurements(output).isEmpty()); + assertEquals("sales", output.get("apiName")); + } + + @Test + void metricNamesAndFieldTypesAreIsolatedAcrossSemanticModels() throws Exception { + Map first = model("first", List.of( + metric("value", "SNOWFLAKE", "SUM(orders.profit)"))); + Map second = model("second", List.of( + metric("value", "ANSI_SQL", "COUNT(orders.profit)"))); + items(items(second, "datasets").get(0), "fields").get(0).put("datatype", "String"); + items(items(second, "datasets").get(0), "fields").get(0).put("dimension", Map.of("is_time", false)); + + List outputs = converter.convert(document(List.of(first, second))); + + assertEquals(2, outputs.size()); + assertEquals("SUM([orders].[profit])", measurements(parse(outputs.get(0))).get(0).get("expression")); + assertEquals("COUNT([orders].[profit])", measurements(parse(outputs.get(1))).get(0).get("expression")); + assertEquals("first", parse(outputs.get(0)).get("apiName")); + assertEquals("second", parse(outputs.get(1)).get("apiName")); + } + + @Test + void fieldsFromAnotherSemanticModelCannotSatisfyAMetricReference() throws Exception { + Map first = model("first", List.of( + metric("valid", "ANSI_SQL", "SUM(orders.profit)"))); + Map second = model("second", List.of( + metric("must_not_leak", "ANSI_SQL", "SUM(orders.profit)"))); + items(second, "datasets").get(0).put("fields", List.of(field("revenue", "Decimal"))); + String input = document(List.of(first, second)); + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("must_not_leak"), error.getMessage()); + assertTrue(error.getMessage().contains("orders.profit"), error.getMessage()); + } + + @Test + void declaredButOmittedCalculatedSqlFieldCannotSatisfyAMetricReference() throws Exception { + Map source = model("sales", List.of()); + Map calculated = field("adjusted", "Decimal"); + calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", "profit__c + 1")))); + items(source, "datasets").get(0).put("fields", List.of(field("profit", "Decimal"), calculated)); + Map output = convertOne(source); + assertEquals(List.of("profit"), items(items(output, "semanticDataObjects").get(0), + "semanticMeasurements").stream().map(item -> item.get("apiName")).toList()); + + source.put("metrics", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); + String input = document(List.of(source)); + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("adjusted_total"), error.getMessage()); + assertTrue(error.getMessage().contains("orders.adjusted"), error.getMessage()); + assertTrue(error.getMessage().contains("not exported"), error.getMessage()); + } + + @Test + void duplicateMetricNamesAreRejectedBeforeExport() throws Exception { + String input = document(List.of(model("sales", List.of( + metric("duplicated", "ANSI_SQL", "SUM(orders.profit)"), + metric("duplicated", "ANSI_SQL", "SUM(orders.revenue)"))))); + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("duplicated"), error.getMessage()); + assertTrue(error.getMessage().contains("duplicate metric"), error.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"AVG(orders.quantity)", "SUM(orders.quantity) / 2", "1.5"}) + void integerMetricDeclarationRejectsFractionalResult(String expression) throws Exception { + Map metric = metric("integer_result", "ANSI_SQL", expression); + metric.put("datatype", "Integer"); + String input = document(List.of(model("sales", List.of(metric)))); + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("integer_result"), error.getMessage()); + assertTrue(error.getMessage().contains("Integer"), error.getMessage()); + assertTrue(error.getMessage().contains("incompatible"), error.getMessage()); + } + + @Test + void integerCountIsExportedAsSalesforceNumber() throws Exception { + Map metric = metric("customer_count", "ANSI_SQL", "COUNT(orders.customer_id)"); + metric.put("datatype", "Integer"); + + Map output = convertOne(model("sales", List.of(metric))); + + assertMeasurementMetadata(measurements(output).get(0)); + assertEquals("COUNT([orders].[customer_id])", measurements(output).get(0).get("expression")); + } + + @Test + void translatedComposedMetricsValidateAgainstTheSalesforceSchema() throws Exception { + assumeTrue(salesforceSchemaExists, "Salesforce schema is required; see README setup instructions"); + Map output = convertOne(model("sales", List.of( + metric("margin", "SNOWFLAKE", "SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0)"), + metric("customers", "ANSI_SQL", "COUNT(DISTINCT orders.customer_id)"), + metric("rounding", "ANSI_SQL", "ROUND(AVG(ABS(orders.profit)), 2) + CEIL(1.2) - FLOOR(1.2)")))); + SchemaValidator validator = new SchemaValidator(JSON, SchemaValidator.SALESFORCE_SCHEMA_PATH); + + for (Map metric : measurements(output)) { + assertMeasurementMetadata(metric); + } + assertDoesNotThrow(() -> validator.validate(output)); + + // OSI expression objects cannot be emitted where Salesforce requires a scalar formula. + Map measurement = measurements(output).get(0); + Object expression = measurement.put("expression", Map.of("dialects", List.of( + dialect("ANSI_SQL", "SUM(orders.profit)")))); + ValidationException error = assertThrows(ValidationException.class, () -> validator.validate(output)); + assertTrue(error.getMessage().contains("expression"), error.getMessage()); + measurement.put("expression", expression); + measurement.put("dataType", "Integer"); + assertThrows(ValidationException.class, () -> validator.validate(output)); + } + + private Map convertOne(Map model) throws IOException { + List outputs = converter.convert(document(List.of(model))); + assertEquals(1, outputs.size()); + return parse(outputs.get(0)); + } + + private static String document(List> models) throws IOException { + return YAML.writeValueAsString(Map.of("version", "0.2.0.dev0", "semantic_model", models)); + } + + private static Map parse(String json) throws IOException { + return JSON.readValue(json, new TypeReference<>() {}); + } + + private static Map model(String name, List> metrics) { + Map model = new LinkedHashMap<>(); + model.put("name", name); + model.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", + "data", "{\"dataspace\":\"default\"}"))); + Map dataset = new LinkedHashMap<>(); + dataset.put("name", "orders"); + dataset.put("source", "orders__dll"); + dataset.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", + "data", "{\"dataObjectType\":\"Dlo\"}"))); + dataset.put("fields", List.of(field("profit", "Decimal"), field("revenue", "Decimal"), + field("quantity", "Integer"), field("customer_id", "String"))); + model.put("datasets", List.of(dataset)); + model.put("metrics", metrics); + return model; + } + + private static Map field(String name, String datatype) { + Map field = new LinkedHashMap<>(); + field.put("name", name); + field.put("datatype", datatype); + field.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", name + "__c")))); + if (datatype.equals("String")) { + field.put("dimension", Map.of("is_time", false)); + } + return field; + } + + private static Map metric(String name, String dialect, String expression) { + Map metric = new LinkedHashMap<>(); + metric.put("name", name); + metric.put("datatype", "Decimal"); + metric.put("expression", Map.of("dialects", List.of(dialect(dialect, expression)))); + return metric; + } + + private static Map dialect(String dialect, String expression) { + return Map.of("dialect", dialect, "expression", expression); + } + + private static void assertMeasurementMetadata(Map measurement) { + assertInstanceOf(String.class, measurement.get("expression")); + assertEquals("Tua", measurement.get("syntax")); + assertEquals("UserAgg", measurement.get("aggregationType")); + assertEquals("Number", measurement.get("dataType")); + } + + private static List> measurements(Map output) { + return items(output, "semanticCalculatedMeasurements"); + } + + @SuppressWarnings("unchecked") + private static List> items(Map object, String key) { + return (List>) object.getOrDefault(key, List.of()); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 627637f0..99cd754f 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -302,9 +302,10 @@ void testMetricsConvertedToSemanticCalculatedMeasurements() throws Exception { assertNotNull(totalRevenue); assertEquals("Sum of all order amounts", totalRevenue.get("description")); assertEquals("Number", totalRevenue.get("dataType")); - // The fixture's metrics only carry an ANSI_SQL dialect (no TABLEAU) -- falls back to - // exporting it unresolved/untranslated rather than failing the whole conversion. + // Legacy ANSI_SQL bracket references are validated and emitted as Tua. assertEquals("SUM([Orders].[amount])", totalRevenue.get("expression")); + assertEquals("Tua", totalRevenue.get("syntax")); + assertEquals("UserAgg", totalRevenue.get("aggregationType")); Map avgOrderValue = calcMeasurements.stream() .filter(m -> "avg_order_value".equals(m.get("apiName"))) @@ -336,7 +337,7 @@ void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { + " - dialect: ANSI_SQL\n" + " expression: SUM([Orders].[amount])\n" + " - dialect: TABLEAU\n" - + " expression: SUM(Orders.amount)\n"); + + " expression: MAX([Orders].[amount])\n"); assertTrue(yamlWithTableauMetric.contains("dialect: TABLEAU"), "fixture text substitution did not match"); List results = converter.convert(yamlWithTableauMetric); @@ -348,7 +349,7 @@ void testMetricExpressionPrefersTableauDialectOverAnsiSql() throws Exception { .findFirst() .orElse(null); assertNotNull(totalRevenue); - assertEquals("SUM(Orders.amount)", totalRevenue.get("expression"), + assertEquals("MAX([Orders].[amount])", totalRevenue.get("expression"), "TABLEAU dialect should be preferred over ANSI_SQL when both are present"); } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java new file mode 100644 index 00000000..7e045fca --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Hand-computed examples evaluated independently from the production parser/AST. + * + *

This is a local regression oracle for the emitted Tua subset, not a Tableau Next execution + * test. In particular, the evaluator models three-valued Boolean logic, null-ignoring aggregates, + * and half-away-from-zero rounding; it does not establish native engine behavior or precision. + */ +class MetricExpressionSemanticsTest { + + private static final List> ORDERS = List.of( + row(10.0, 2.0, true), + row(10.0, 0.0, false), + row(-4.0, 4.0, null), + row(null, null, null)); + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void aggregatesIgnoreNullsButPreserveDuplicates(String dialect) { + assertValue(dialect, "SUM(orders.amount)", ORDERS, 16.0); + assertValue(dialect, "AVG(orders.amount)", ORDERS, 16.0 / 3); + assertValue(dialect, "MIN(orders.amount)", ORDERS, -4.0); + assertValue(dialect, "MAX(orders.amount)", ORDERS, 10.0); + assertValue(dialect, "COUNT(orders.amount)", ORDERS, 3.0); + assertValue(dialect, "COUNT(DISTINCT orders.amount)", ORDERS, 2.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void emptyAndAllNullInputsHaveDifferentCountAndSumResults(String dialect) { + for (List> rows : List.of( + List.>of(), List.of(row(null, null, null)))) { + for (String aggregate : List.of("SUM", "AVG", "MIN", "MAX")) { + assertValue(dialect, aggregate + "(orders.amount)", rows, null); + } + assertValue(dialect, "COUNT(orders.amount)", rows, 0.0); + assertValue(dialect, "COUNT(DISTINCT orders.amount)", rows, 0.0); + assertValue(dialect, "COALESCE(SUM(orders.amount), 0)", rows, 0.0); + } + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void composedConditionalArithmeticPreservesNullsAndGrouping(String dialect) { + // Per-row numerator contributions are 20, 10, -4, 0; COUNT ignores the null amount. + assertValue(dialect, + "SUM(CASE WHEN orders.flag AND orders.amount > 0 " + + "THEN orders.amount * 2 ELSE COALESCE(orders.amount, 0) END) " + + "/ NULLIF(COUNT(orders.amount), 0)", + ORDERS, 26.0 / 3); + assertValue(dialect, + "(SUM(orders.amount) + 2) * (MAX(orders.cost) - MIN(orders.cost))", + ORDERS, 72.0); + assertValue(dialect, + "SUM(CASE WHEN orders.amount < 0 THEN -orders.amount END)", + ORDERS, 4.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void guardedRatiosReturnNullForZeroAndEmptyDenominators(String dialect) { + String ratio = "SUM(orders.amount) / NULLIF(SUM(orders.cost), 0)"; + assertValue(dialect, ratio, ORDERS, 16.0 / 6); + assertValue(dialect, ratio, List.of(row(8.0, 1.0, true), row(4.0, -1.0, false)), null); + assertValue(dialect, ratio, List.of(row(8.0, null, true)), null); + assertValue(dialect, ratio, List.of(), null); + assertValue(dialect, "COALESCE(" + ratio + ", 0)", List.of(), 0.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void nestedCoalesceSelectsTheFirstNonNullValueIncludingZero(String dialect) { + String expression = "COALESCE(SUM(orders.amount), NULLIF(SUM(orders.cost), 0), 7)"; + assertValue(dialect, expression, ORDERS, 16.0); + assertValue(dialect, expression, List.of(row(0.0, 9.0, true)), 0.0); + assertValue(dialect, expression, List.of(row(null, 9.0, true)), 9.0); + assertValue(dialect, expression, List.of(row(null, 0.0, true)), 7.0); + assertValue(dialect, expression, List.of(), 7.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void nullablePredicatesUseThreeValuedLogic(String dialect) { + assertValue(dialect, + "SUM(CASE WHEN NOT (orders.flag = TRUE) THEN orders.amount ELSE 0 END)", + ORDERS, 10.0); + assertValue(dialect, + "SUM(CASE WHEN orders.amount IS NOT NULL AND " + + "(orders.flag = FALSE OR orders.flag IS NULL) " + + "THEN orders.amount ELSE 0 END)", + ORDERS, 6.0); + assertValue(dialect, + "SUM(CASE WHEN NOT (orders.flag OR orders.amount < 0) THEN 1 ELSE 0 END)", + ORDERS, 1.0); + assertValue(dialect, + "SUM(CASE WHEN orders.flag IS NULL THEN " + + "CASE WHEN orders.amount IS NULL THEN 3 ELSE 2 END ELSE 0 END)", + ORDERS, 5.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void numericFunctionsComposeAroundNegativeAggregates(String dialect) { + List> negative = List.of(row(-2.55, 0.0, true), row(-2.55, 0.0, false)); + assertValue(dialect, "ROUND(AVG(orders.amount), 1)", negative, -2.6); + assertValue(dialect, "ABS(ROUND(AVG(orders.amount), 1))", negative, 2.6); + assertValue(dialect, "CEIL(AVG(orders.amount)) + FLOOR(AVG(orders.amount))", negative, -5.0); + assertValue(dialect, "ABS(ROUND(AVG(orders.amount), 1))", List.of(), null); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void textPredicatesPreserveDuplicatesNullsAndEscapedApostrophes(String dialect) { + List> orders = List.of( + row(10.0, 0.0, true, "paid"), + row(7.0, 0.0, true, "paid"), + row(4.0, 0.0, false, "pending"), + row(9.0, 0.0, null, null), + row(2.0, 0.0, true, "O'Brien")); + assertValue(dialect, + "SUM(CASE WHEN orders.status = 'paid' THEN orders.amount ELSE 0 END)", + orders, 17.0); + assertValue(dialect, "COUNT(DISTINCT orders.status)", orders, 3.0); + assertValue(dialect, "COUNT(orders.status)", orders, 4.0); + assertValue(dialect, + "SUM(CASE WHEN COALESCE(NULLIF(orders.status, 'pending'), 'missing') = 'missing' " + + "THEN orders.amount ELSE 0 END)", + orders, 13.0); + assertValue(dialect, + "SUM(CASE WHEN COALESCE(orders.status, 'paid') = 'paid' THEN orders.amount ELSE 0 END)", + orders, 26.0); + assertValue(dialect, + "SUM(CASE WHEN orders.status = 'O''Brien' THEN orders.amount ELSE 0 END)", + orders, 2.0); + assertValue(dialect, "COUNT(DISTINCT orders.status)", List.of(), 0.0); + assertValue(dialect, "COUNT(DISTINCT orders.status)", List.of(row(null, null, null, null)), 0.0); + } + + private static void assertValue( + String dialect, String sql, List> rows, Double expected) { + Map metric = Map.of( + "name", "fixture_metric", + "datatype", "Decimal", + "expression", Map.of("dialects", List.of(Map.of("dialect", dialect, "expression", sql)))); + Map source = Map.of("datasets", List.of(Map.of( + "name", "orders", + "fields", List.of( + Map.of("name", "amount", "datatype", "Decimal"), + Map.of("name", "cost", "datatype", "Decimal"), + Map.of("name", "flag", "datatype", "Boolean"), + Map.of("name", "status", "datatype", "String"))))); + Map target = Map.of("semanticDataObjects", List.of(Map.of( + "apiName", "orders", + "semanticMeasurements", List.of( + Map.of("apiName", "amount", "dataType", "Number"), + Map.of("apiName", "cost", "dataType", "Number")), + "semanticDimensions", List.of( + Map.of("apiName", "flag", "dataType", "Boolean"), + Map.of("apiName", "status", "dataType", "Text"))))); + MetricExpressionTranslator.Result translated = MetricExpressionTranslator.translate(metric, source, target); + assertEquals("Number", translated.dataType(), sql); + Object actual = new TuaSubsetEvaluator(translated.expression()).evaluate(rows); + String message = dialect + ": " + sql + " -> " + translated.expression(); + if (expected == null) { + assertNull(actual, message); + } else { + assertInstanceOf(Number.class, actual, message); + assertEquals(expected, ((Number) actual).doubleValue(), 1e-12, message); + } + } + + private static Map row(Double amount, Double cost, Boolean flag) { + return row(amount, cost, flag, null); + } + + private static Map row(Double amount, Double cost, Boolean flag, String status) { + Map result = new LinkedHashMap<>(); + result.put("amount", amount); + result.put("cost", cost); + result.put("flag", flag); + result.put("status", status); + return result; + } + + /** Reads only generated Tua; it neither accepts SQL CASE/NULLIF nor uses production AST nodes. */ + private static final class TuaSubsetEvaluator { + private static final Pattern TOKEN = Pattern.compile( + "\\s*(\\[[^\\]]+\\]|[0-9]+(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?" + + "|'(?:[^']|'')*'|[A-Za-z_][A-Za-z_0-9]*|<>|!=|<=|>=|[().,+*/=<>-])"); + private final List tokens = new ArrayList<>(); + private int position; + + TuaSubsetEvaluator(String expression) { + Matcher matcher = TOKEN.matcher(expression); + int offset = 0; + while (offset < expression.length()) { + if (expression.substring(offset).isBlank()) { + break; + } + if (!matcher.find(offset) || matcher.start() != offset) { + throw new AssertionError("Unexpected Tua token: " + expression.substring(offset)); + } + tokens.add(matcher.group(1)); + offset = matcher.end(); + } + } + + Object evaluate(List> rows) { + Calculation calculation = expression(0); + assertEquals(tokens.size(), position, "Unconsumed generated Tua tokens"); + return calculation.value(rows, Map.of()); + } + + private Calculation expression(int minimum) { + Calculation left = prefix(); + while (position < tokens.size() && precedence(tokens.get(position)) >= minimum) { + String operator = next().toUpperCase(java.util.Locale.ROOT); + Calculation right = expression(precedence(operator) + 1); + Calculation previous = left; + left = (rows, row) -> binary(operator, previous.value(rows, row), right.value(rows, row)); + } + return left; + } + + private Calculation prefix() { + String token = next(); + if (token.equals("(")) { + Calculation result = expression(0); + expect(")"); + return result; + } + if (token.equalsIgnoreCase("IF")) { + Calculation condition = expression(0); + expect("THEN"); + Calculation yes = expression(0); + expect("ELSE"); + Calculation no = expression(0); + expect("END"); + return (rows, row) -> (Boolean.TRUE.equals(condition.value(rows, row)) ? yes : no).value(rows, row); + } + if (token.equalsIgnoreCase("NOT") || token.equals("-")) { + Calculation child = expression(token.equals("-") ? 7 : 3); + return (rows, row) -> { + Object value = child.value(rows, row); + return value == null ? null : token.equals("-") ? -number(value) : !(Boolean) value; + }; + } + if (token.equalsIgnoreCase("NULL")) { + return (rows, row) -> null; + } + if (token.equalsIgnoreCase("TRUE") || token.equalsIgnoreCase("FALSE")) { + return (rows, row) -> Boolean.valueOf(token); + } + if (token.startsWith("'")) { + String literal = token.substring(1, token.length() - 1).replace("''", "'"); + return (rows, row) -> literal; + } + if (token.startsWith("[")) { + assertEquals("[orders]", token); + expect("."); + String field = next(); + assertTrue(field.startsWith("[") && field.endsWith("]")); + String name = field.substring(1, field.length() - 1); + return (rows, row) -> { + assertTrue(row.containsKey(name), "Unknown generated field: " + name); + return row.get(name); + }; + } + if (Character.isDigit(token.charAt(0))) { + return (rows, row) -> Double.valueOf(token); + } + expect("("); + List arguments = new ArrayList<>(); + do { + arguments.add(expression(0)); + } while (take(",")); + expect(")"); + return function(token.toUpperCase(java.util.Locale.ROOT), arguments); + } + + private static Calculation function(String name, List arguments) { + if (List.of("SUM", "AVG", "MIN", "MAX", "COUNT", "COUNTD").contains(name)) { + assertEquals(1, arguments.size()); + return (rows, row) -> { + List values = rows.stream() + .map(input -> arguments.getFirst().value(rows, input)) + .filter(java.util.Objects::nonNull).toList(); + if (name.equals("COUNT")) { + return (double) values.size(); + } + if (name.equals("COUNTD")) { + return (double) values.stream().distinct().count(); + } + if (values.isEmpty()) { + return null; + } + return switch (name) { + case "SUM" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).sum(); + case "AVG" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).average().orElseThrow(); + case "MIN" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).min().orElseThrow(); + case "MAX" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).max().orElseThrow(); + default -> throw new AssertionError(name); + }; + }; + } + assertTrue(List.of("IFNULL", "ISNULL", "ABS", "CEILING", "FLOOR", "ROUND").contains(name), + "Unsupported generated function: " + name); + return (rows, row) -> { + Object value = arguments.getFirst().value(rows, row); + if (name.equals("IFNULL")) { + assertEquals(2, arguments.size()); + return value != null ? value : arguments.get(1).value(rows, row); + } + if (name.equals("ISNULL")) { + assertEquals(1, arguments.size()); + return value == null; + } + if (value == null) { + return null; + } + return switch (name) { + case "ABS" -> Math.abs(number(value)); + case "CEILING" -> Math.ceil(number(value)); + case "FLOOR" -> Math.floor(number(value)); + case "ROUND" -> BigDecimal.valueOf(number(value)).setScale( + arguments.size() == 1 ? 0 : (int) number(arguments.get(1).value(rows, row)), + RoundingMode.HALF_UP).doubleValue(); + default -> throw new AssertionError("Unsupported generated function: " + name); + }; + }; + } + + private static Object binary(String operator, Object left, Object right) { + if (operator.equals("AND")) { + if (Boolean.FALSE.equals(left) || Boolean.FALSE.equals(right)) { + return false; + } + return left == null || right == null ? null : Boolean.TRUE; + } + if (operator.equals("OR")) { + if (Boolean.TRUE.equals(left) || Boolean.TRUE.equals(right)) { + return true; + } + return left == null || right == null ? null : Boolean.FALSE; + } + if (left == null || right == null) { + return null; + } + return switch (operator) { + case "+" -> number(left) + number(right); + case "-" -> number(left) - number(right); + case "*" -> number(left) * number(right); + case "/" -> { + assertNotEquals(0.0, number(right), "Generated expression evaluated an unguarded zero divisor"); + yield number(left) / number(right); + } + case "=" -> left.equals(right); + case "!=", "<>" -> !left.equals(right); + case "<" -> number(left) < number(right); + case "<=" -> number(left) <= number(right); + case ">" -> number(left) > number(right); + case ">=" -> number(left) >= number(right); + default -> throw new AssertionError(operator); + }; + } + + private static double number(Object value) { + return ((Number) value).doubleValue(); + } + + private static int precedence(String token) { + return switch (token.toUpperCase(java.util.Locale.ROOT)) { + case "OR" -> 1; + case "AND" -> 2; + case "=", "!=", "<>", "<", ">", "<=", ">=" -> 4; + case "+", "-" -> 5; + case "*", "/" -> 6; + default -> -1; + }; + } + + private boolean take(String token) { + if (position < tokens.size() && tokens.get(position).equalsIgnoreCase(token)) { + position++; + return true; + } + return false; + } + + private String next() { + assertTrue(position < tokens.size(), "Unexpected end of generated Tua expression"); + return tokens.get(position++); + } + + private void expect(String token) { + assertEquals(token, next().toUpperCase(java.util.Locale.ROOT)); + } + } + + @FunctionalInterface + private interface Calculation { + Object value(List> rows, Map row); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java new file mode 100644 index 00000000..79d9d832 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +class MetricExpressionTranslatorTest { + private static final Map TYPES = Map.of( + "amount", "Decimal", "profit", "Decimal", "revenue", "Decimal", "discount", "Decimal", + "quantity", "Integer", "status", "String", "active", "Boolean", "ordered", "Date"); + + private static Map source() { + return Map.of("datasets", List.of(Map.of("name", "orders", "fields", TYPES.entrySet().stream() + .map(entry -> Map.of("name", entry.getKey(), "datatype", entry.getValue())).toList()))); + } + + private static Map target() { + return Map.of("semanticDataObjects", List.of(Map.of("apiName", "orders", "semanticMeasurements", + TYPES.entrySet().stream().map(entry -> Map.of("apiName", entry.getKey(), "dataType", + SalesforceDataTypeMapper.toSalesforce(entry.getValue()))).toList()))); + } + + private static Map metric(String dialect, String expression) { + return new LinkedHashMap<>(Map.of("name", "net_value", "expression", Map.of("dialects", + List.of(Map.of("dialect", dialect, "expression", expression))))); + } + + private static String translate(String dialect, String expression) { + var result = MetricExpressionTranslator.translate(metric(dialect, expression), source(), target()); + assertEquals("Number", result.dataType()); + return result.expression(); + } + + static Stream supported() { + return Stream.of( + Arguments.of("SUM(orders.amount)", "SUM([orders].[amount])"), + Arguments.of("AVG(amount)", "AVG([orders].[amount])"), + Arguments.of("MIN(ORDERS.amount)", "MIN([orders].[amount])"), + Arguments.of("MAX(orders.amount)", "MAX([orders].[amount])"), + Arguments.of("COUNT(orders.status)", "COUNT([orders].[status])"), + Arguments.of("COUNT(DISTINCT orders.status)", "COUNTD([orders].[status])"), + Arguments.of("SUM(orders.amount * (1 - orders.discount))", + "SUM(([orders].[amount] * (1 - [orders].[discount])))"), + Arguments.of("SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0)", + "(SUM([orders].[profit]) / (IF (SUM([orders].[revenue]) = 0) THEN NULL ELSE SUM([orders].[revenue]) END))"), + Arguments.of("SUM(CASE WHEN orders.status = 'paid' THEN orders.amount ELSE 0 END)", + "SUM((IF ([orders].[status] = 'paid') THEN [orders].[amount] ELSE 0 END))"), + Arguments.of("COALESCE(SUM(orders.amount), AVG(orders.revenue), 0)", + "IFNULL(SUM([orders].[amount]), IFNULL(AVG([orders].[revenue]), 0))"), + Arguments.of("SUM(CASE WHEN orders.amount IS NOT NULL THEN orders.amount END)", + "SUM((IF (NOT ISNULL([orders].[amount])) THEN [orders].[amount] ELSE NULL END))"), + Arguments.of("ROUND(AVG(ABS(orders.amount)), 2)", "ROUND(AVG(ABS([orders].[amount])), 2)"), + Arguments.of("SUM(CEIL(orders.amount) - FLOOR(orders.amount))", + "SUM((CEILING([orders].[amount]) - FLOOR([orders].[amount])))"), + Arguments.of("1 + 2 * 3 - 4 / 2", "((1 + (2 * 3)) - (4 / 2))"), + Arguments.of(".25 + 1e2", "(0.25 + 100)"), + Arguments.of("SUM(orders.quantity) + -2", "(SUM([orders].[quantity]) + (-2))"), + Arguments.of("CASE WHEN MAX(orders.status) = 'z' THEN 1 ELSE 0 END", + "(IF (MAX([orders].[status]) = 'z') THEN 1 ELSE 0 END)") + ); + } + + @ParameterizedTest + @MethodSource("supported") + void compilesComposedSqlInBothDialects(String expression, String expected) { + assertEquals(expected, translate("SNOWFLAKE", expression)); + assertEquals(expected, translate("ANSI_SQL", expression)); + // Every generated expression can be read through the bounded TABLEAU path. + assertEquals(expected, translate("TABLEAU", expected)); + } + + @Test + void notUsesSqlPrecedenceAndPreservesThreeValuedPredicates() { + assertEquals("SUM((IF ((NOT ([orders].[amount] = 0)) OR ([orders].[active] AND ISNULL([orders].[discount])))" + + " THEN 1 ELSE 0 END))", + translate("SNOWFLAKE", "SUM(CASE WHEN NOT orders.amount = 0 OR orders.active AND orders.discount IS NULL THEN 1 ELSE 0 END)")); + } + + @Test + void nestedCasesAndElseifRetainBranchOrder() { + String sql = "SUM(CASE WHEN orders.active THEN CASE WHEN orders.amount > 0 THEN 2 ELSE 3 END WHEN orders.status = 'paid' THEN 4 END)"; + String expected = "SUM((IF [orders].[active] THEN (IF ([orders].[amount] > 0) THEN 2 ELSE 3 END)" + + " ELSEIF ([orders].[status] = 'paid') THEN 4 ELSE NULL END))"; + assertEquals(expected, translate("SNOWFLAKE", sql)); + assertEquals(expected, translate("TABLEAU", expected)); + } + + @Test + void identifiersAreBoundToSemanticNamesAndStringContentsAreNotReferences() { + assertEquals("SUM([orders].[amount])", translate("SNOWFLAKE", "SUM(\"ORDERS\".\"AMOUNT\")")); + assertEquals("SUM([orders].[amount])", translate("ANSI_SQL", "SUM([orders].[amount])")); + assertEquals("SUM((IF ([orders].[status] = 'missing.field O''Brien') THEN 1 ELSE 0 END))", + translate("SNOWFLAKE", "SUM(CASE WHEN orders.status = 'missing.field O''Brien' THEN 1 ELSE 0 END)")); + assertEquals("SUM((IF ([orders].[status] = 'paid') THEN 1 ELSE 0 END))", + translate("TABLEAU", "SUM(IF [orders].[status] = \"paid\" THEN 1 ELSE 0 END)")); + } + + static Stream unsupported() { + return Stream.of( + Arguments.of("SUM(orders.missing)", "Unknown field"), + Arguments.of("SUM(missing.amount)", "Unknown dataset"), + Arguments.of("SUM(db.orders.amount)", "physical source paths"), + Arguments.of("SUM(orders.amount__c)", "Unknown field"), + Arguments.of("SUM(orders.amount) + orders.amount", "mix aggregate"), + Arguments.of("CASE WHEN orders.active THEN SUM(orders.amount) ELSE 0 END", "mix aggregate"), + Arguments.of("SUM(AVG(orders.amount))", "nested aggregate"), + Arguments.of("SUM(orders.status)", "numeric"), + Arguments.of("AVG(orders.active)", "numeric"), + Arguments.of("SUM(CASE WHEN orders.amount THEN 1 ELSE 0 END)", "BOOLEAN"), + Arguments.of("SUM(CASE WHEN orders.active THEN orders.amount ELSE 'zero' END)", "incompatible types"), + Arguments.of("COALESCE(SUM(orders.amount), '0')", "incompatible types"), + Arguments.of("NULLIF(SUM(orders.amount), '0')", "incompatible types"), + Arguments.of("SUM(orders.amount) AND TRUE", "BOOLEAN"), + Arguments.of("SUM(CASE WHEN orders.active > TRUE THEN 1 ELSE 0 END)", "ordered comparison"), + Arguments.of("orders.amount + 1", "unaggregated"), + Arguments.of("MAX(orders.status)", "must be numeric"), + Arguments.of("TRUE", "must be numeric"), + Arguments.of("COUNT(*)", "COUNT(*)"), + Arguments.of("COUNT(1)", "declared field"), + Arguments.of("COUNT(orders.quantity + 1)", "counting expressions"), + Arguments.of("SUM(1)", "establish its dataset"), + Arguments.of("SUM(DISTINCT orders.amount)", "DISTINCT"), + Arguments.of("COUNT(DISTINCT orders.amount, orders.quantity)", "expects 1"), + Arguments.of("ROUND(SUM(orders.amount), 2, 'HALF_TO_EVEN')", "expects 1 to 2"), + Arguments.of("ROUND(SUM(orders.amount), .5)", "integer literal"), + Arguments.of("ROUND(SUM(orders.amount), orders.quantity)", "integer literal"), + Arguments.of("CEIL(SUM(orders.amount), 2)", "expects 1"), + Arguments.of("FLOOR(SUM(orders.amount), 2)", "expects 1"), + Arguments.of("ABS()", "expects 1"), + Arguments.of("COALESCE(SUM(orders.amount))", "expects 2"), + Arguments.of("NULLIF(SUM(orders.amount))", "expects 2"), + Arguments.of("MEDIAN(orders.amount)", "unsupported function"), + Arguments.of("CAST(orders.amount AS DECIMAL)", "expected )"), + Arguments.of("SUM(YEAR(orders.ordered))", "unsupported function"), + Arguments.of("SUM(LENGTH(orders.status))", "unsupported function"), + Arguments.of("SUM(orders.amount) OVER ()", "unexpected token"), + Arguments.of("SUM(orders.amount) FILTER (WHERE orders.active)", "unexpected token"), + Arguments.of("{ FIXED : SUM(orders.amount) }", "unsupported character"), + Arguments.of("SELECT SUM(orders.amount)", "Unknown field"), + Arguments.of("SUM(orders.amount);", "unsupported character"), + Arguments.of("SUM(orders.amount) -- comment", "comments"), + Arguments.of("SUM(orders.amount) /* comment */", "comments"), + Arguments.of("SUM(orders.amount", "expected )"), + Arguments.of("SUM(orders.amount) trailing", "unexpected token"), + Arguments.of("1.2.3", "invalid numeric"), + Arguments.of("1e", "invalid numeric"), + Arguments.of("1e1000000000", "too large"), + Arguments.of("SUM(CASE WHEN orders.status = 'unterminated THEN 1 END)", "unterminated"), + Arguments.of("SUM(orders.amount) % 2", "unsupported character"), + Arguments.of("SUM(orders.amount) ^ 2", "unsupported character"), + Arguments.of("SUM(orders.amount) || 'x'", "unsupported character"), + Arguments.of("SUM(orders.amount) / 0", "division by literal zero"), + Arguments.of("net_value + 1", "Unknown field") + ); + } + + @ParameterizedTest + @MethodSource("unsupported") + void rejectsWithMetricNameAndActionableReason(String expression, String reason) { + ConversionException exception = assertThrows(ConversionException.class, + () -> translate("SNOWFLAKE", expression)); + assertTrue(exception.getMessage().contains("Metric 'net_value'"), exception.getMessage()); + assertTrue(exception.getMessage().contains(reason), exception.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"SUM(orders.amount)", "SUM([orders].[missing])", "COUNT(DISTINCT [orders].[amount])", + "COALESCE(SUM([orders].[amount]), 0)", "CASE WHEN TRUE THEN 1 ELSE 0 END", "not a formula", + "SUM([orders].[amount]) OVER ()", "SUM([orders].[amount] + [orders].[status])"}) + void tableauCannotBypassParsingOrReferenceValidation(String expression) { + assertThrows(ConversionException.class, () -> translate("TABLEAU", expression)); + } + + @Test + void selectedDialectFailureNeverFallsBack() { + Map metric = metric("TABLEAU", "SUM([orders].[missing])"); + metric.put("expression", Map.of("dialects", List.of( + Map.of("dialect", "ANSI_SQL", "expression", "SUM(orders.amount)"), + Map.of("dialect", "TABLEAU", "expression", "SUM([orders].[missing])")))); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, source(), target())); + } + + @Test + void rejectsDuplicateDialectAndMissingOrEmptyExpressions() { + Map metric = metric("SNOWFLAKE", "SUM(orders.amount)"); + Map entry = Map.of("dialect", "SNOWFLAKE", "expression", "SUM(orders.amount)"); + metric.put("expression", Map.of("dialects", List.of(entry, entry))); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, source(), target())); + for (Object expression : List.of(Map.of(), Map.of("dialects", List.of()), + Map.of("dialects", List.of(Map.of("dialect", "BIGQUERY", "expression", "1"))))) { + metric.put("expression", expression); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, source(), target())); + } + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", " ")); + } + + @Test + void outputDatatypeMustAgreeWithTheFormula() { + Map metric = metric("SNOWFLAKE", "AVG(orders.quantity)"); + for (String datatype : List.of("Integer", "String", "Boolean", "Date", "Opaque", "Time")) { + metric.put("datatype", datatype); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, source(), target())); + } + metric.put("datatype", "Decimal"); + assertEquals("Number", MetricExpressionTranslator.translate(metric, source(), target()).dataType()); + Map countMetric = metric("SNOWFLAKE", "COUNT(orders.quantity)"); + countMetric.put("datatype", "Integer"); + assertEquals("Number", MetricExpressionTranslator.translate(countMetric, source(), target()).dataType()); + Map nullMetric = metric("SNOWFLAKE", "NULL"); + nullMetric.put("datatype", "Decimal"); + assertEquals("NULL", MetricExpressionTranslator.translate(nullMetric, source(), target()).expression()); + } + + @Test + void limitsNestingAndExpansionWithoutStackOverflow() { + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "(".repeat(200) + "1" + ")".repeat(200))); + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "-".repeat(200) + "1")); + String formula = "SUM(orders.amount)"; + for (int i = 0; i < 20; i++) formula = "NULLIF(" + formula + ", 0)"; + String expanded = formula; + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", expanded)); + } + + @Test + void temporalAggregatesCanBeUsedInNumericPredicates() { + assertEquals("(IF (MIN([orders].[ordered]) = MAX([orders].[ordered])) THEN 1 ELSE 0 END)", + translate("SNOWFLAKE", "CASE WHEN MIN(orders.ordered) = MAX(orders.ordered) THEN 1 ELSE 0 END")); + } + + @Test + void separateAggregatesRequireConnectedDatasets() { + Map twoSources = Map.of("datasets", List.of( + Map.of("name", "orders", "fields", List.of(Map.of("name", "amount", "datatype", "Decimal"))), + Map.of("name", "returns", "fields", List.of(Map.of("name", "amount", "datatype", "Decimal"))))); + Map twoTargets = new LinkedHashMap<>(Map.of("semanticDataObjects", List.of( + Map.of("apiName", "orders", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number"))), + Map.of("apiName", "returns", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number")))))); + Map metric = metric("SNOWFLAKE", "SUM(orders.amount) - SUM(returns.amount)"); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, twoSources, twoTargets)); + twoTargets.put("semanticRelationships", List.of(Map.of("leftSemanticDefinitionApiName", "orders", + "rightSemanticDefinitionApiName", "returns"))); + assertEquals("(SUM([orders].[amount]) - SUM([returns].[amount]))", + MetricExpressionTranslator.translate(metric, twoSources, twoTargets).expression()); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate( + metric("SNOWFLAKE", "SUM(orders.amount - returns.amount)"), twoSources, twoTargets)); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java new file mode 100644 index 00000000..fa6c4c8f --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ossie.converter.MetricFieldResolver.Identifier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class MetricFieldResolverTest { + + @Test + void acceptsDatasetsConnectedByAnExportedRelationshipInEitherDirection() { + MetricFieldResolver resolver = graphResolver(List.of( + relationship("Orders", "Customers")), "Orders", "Customers"); + resolver.validateDatasets(Set.of("Orders", "Customers")); + resolver.validateDatasets(Set.of("Orders")); + resolver.validateDatasets(Set.of()); + } + + @Test + void acceptsFactDatasetsConnectedThroughAnIntermediateSharedDimension() { + MetricFieldResolver resolver = graphResolver(List.of( + relationship("Orders", "Customers"), relationship("Returns", "Customers")), + "Orders", "Returns", "Customers"); + resolver.validateDatasets(Set.of("Orders", "Returns")); + } + + @Test + void rejectsDisconnectedDatasetsAndRelationshipsThroughMissingTargets() { + MetricFieldResolver resolver = graphResolver(List.of( + relationship("Orders", "MissingCustomers"), relationship("Returns", "MissingCustomers")), + "Orders", "Returns"); + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> resolver.validateDatasets(Set.of("Orders", "Returns"))); + assertTrue(error.getMessage().contains("disconnected datasets Orders, Returns"), error.getMessage()); + assertTrue(error.getMessage().contains("declare supported relationships"), error.getMessage()); + } + + @Test + void bindsQualifiedAndUniqueUnqualifiedSqlNamesToExportedApiNames() { + MetricFieldResolver resolver = resolver("Orders", "revenue", "Decimal", "Currency"); + assertEquals("[Orders].[revenue]", resolver.resolve(sql("oRders", "REVENUE"), false).expression()); + assertEquals("[Orders].[revenue]", resolver.resolve(sql("revenue"), false).expression()); + assertEquals("Orders", resolver.resolve(sql("revenue"), false).dataset()); + assertEquals("Decimal", resolver.resolve(sql("revenue"), false).datatype()); + } + + @Test + void normalizesQuotedSqlIdentifiersAsSpecified() { + MetricFieldResolver resolver = resolver("orders", "revenue", "Decimal", "Number"); + assertEquals("[orders].[revenue]", resolver.resolve( + List.of(new Identifier("ORDERS", true), new Identifier("REVENUE", true)), false).expression()); + assertError(resolver, List.of(new Identifier("orders", true), new Identifier("revenue", true)), + false, "Unknown dataset"); + } + + @Test + void resolvesQuotedDeclarationsWithoutRenamingExportedObjects() { + MetricFieldResolver resolver = resolver("\"Order Items\"", "\"Unit \"\"Price\"\"\"", "Decimal", "Number"); + assertEquals("[\"Order Items\"].[\"Unit \"\"Price\"\"\"]", resolver.resolve( + List.of(new Identifier("Order Items", true), new Identifier("Unit \"Price\"", true)), false) + .expression()); + assertError(resolver, sql("ORDER ITEMS", "UNIT PRICE"), false, "Unknown dataset"); + } + + @Test + void tableauReferencesUseExactApiNamesAndRequireDataset() { + MetricFieldResolver resolver = resolver("Orders", "revenue", "Integer", "Number"); + assertEquals("[Orders].[revenue]", resolver.resolve(sql("Orders", "revenue"), true).expression()); + assertError(resolver, sql("orders", "revenue"), true, "Unknown dataset"); + assertError(resolver, sql("Orders", "Revenue"), true, "Unknown field"); + assertError(resolver, sql("revenue"), true, "must use [dataset].[field]"); + } + + @Test + void rejectsPhysicalSourceNamesAndUndeclaredFields() { + MetricFieldResolver resolver = resolver("Orders", "revenue", "Decimal", "Number"); + assertError(resolver, sql("Orders__dll", "revenue"), false, "Unknown dataset"); + assertError(resolver, sql("Orders", "revenue__c"), false, "Unknown field"); + assertError(resolver, sql("warehouse", "Orders", "revenue"), false, "physical source paths"); + assertError(resolver, List.of(), false, "must name a declared field"); + } + + @Test + void rejectsAmbiguousUnqualifiedFieldsAcrossDatasets() { + Map orders = dataset("Orders", field("amount", "Decimal")); + Map returns = dataset("Returns", field("amount", "Decimal")); + MetricFieldResolver resolver = new MetricFieldResolver( + Map.of("datasets", List.of(orders, returns)), + Map.of("semanticDataObjects", List.of(targetDataset("Orders", targetField("amount", "Number")), + targetDataset("Returns", targetField("amount", "Number"))))); + assertError(resolver, sql("amount"), false, "Ambiguous field"); + assertEquals("[Orders].[amount]", resolver.resolve(sql("Orders", "amount"), false).expression()); + } + + @Test + void rejectsDuplicateDatasetDeclarationsEvenWhenOnlyOneHasTheField() { + MetricFieldResolver resolver = new MetricFieldResolver(Map.of("datasets", List.of( + dataset("Orders", field("amount", "Decimal")), dataset("ORDERS", field("other", "Decimal")))), + Map.of()); + assertError(resolver, sql("Orders", "amount"), false, "Ambiguous dataset"); + assertError(resolver, sql("amount"), false, "Ambiguous dataset"); + } + + @Test + void rejectsDuplicateFieldDeclarations() { + MetricFieldResolver resolver = new MetricFieldResolver(Map.of("datasets", List.of( + dataset("Orders", field("amount", "Decimal"), field("AMOUNT", "Decimal")))), Map.of()); + assertError(resolver, sql("Orders", "amount"), false, "Ambiguous field"); + } + + @Test + void checksThatDeclaredFieldsWereActuallyExportedAsDirectFields() { + Map source = Map.of("datasets", List.of(dataset("Orders", field("amount", "Decimal")))); + MetricFieldResolver missingDataset = new MetricFieldResolver(source, Map.of()); + assertError(missingDataset, sql("Orders", "amount"), false, "was not exported"); + + MetricFieldResolver calculatedField = new MetricFieldResolver(source, Map.of( + "semanticDataObjects", List.of(targetDataset("Orders")), + "semanticCalculatedDimensions", List.of(Map.of("apiName", "amount", "expression", "1 + 2")))); + assertError(calculatedField, sql("Orders", "amount"), false, "calculated or omitted fields"); + } + + @Test + void rejectsDuplicateExportedObjectsAndFieldsAcrossKinds() { + Map source = Map.of("datasets", List.of(dataset("Orders", field("amount", "Decimal")))); + Map target = targetDataset("Orders", targetField("amount", "Number")); + MetricFieldResolver duplicateDatasets = new MetricFieldResolver(source, + Map.of("semanticDataObjects", List.of(target, target))); + assertError(duplicateDatasets, sql("amount"), false, "Ambiguous exported Salesforce dataset"); + + Map duplicateFields = Map.of("apiName", "Orders", + "semanticDimensions", List.of(targetField("amount", "Number")), + "semanticMeasurements", List.of(targetField("amount", "Number"))); + MetricFieldResolver resolver = new MetricFieldResolver(source, + Map.of("semanticDataObjects", List.of(duplicateFields))); + assertError(resolver, sql("amount"), false, "Ambiguous exported Salesforce field"); + } + + @Test + void infersMissingPortableTypeOnlyFromKnownExportedType() { + MetricFieldResolver resolver = new MetricFieldResolver( + Map.of("datasets", List.of(dataset("Orders", Map.of("name", "amount")))), + Map.of("semanticDataObjects", List.of(targetDataset("Orders", targetField("amount", "Currency"))))); + assertEquals("Decimal", resolver.resolve(sql("amount"), false).datatype()); + } + + @Test + void rejectsUnknownOrConflictingTypes() { + assertError(resolver("Orders", "amount", "String", "Number"), sql("amount"), false, + "use compatible field types"); + assertError(resolver("Orders", "amount", "Opaque", "Geo"), sql("amount"), false, + "no supported datatype"); + MetricFieldResolver resolver = new MetricFieldResolver( + Map.of("datasets", List.of(dataset("Orders", Map.of("name", "amount")))), + Map.of("semanticDataObjects", List.of(targetDataset("Orders", targetField("amount", "Geo"))))); + assertError(resolver, sql("amount"), false, "no supported datatype"); + } + + @ParameterizedTest + @ValueSource(strings = {"bad[field", "bad]field", "bad\nfield", "bad\tfield", "bad\u0000field"}) + void rejectsUnrepresentableExportedNames(String name) { + MetricFieldResolver resolver = resolver("Orders", name, "Decimal", "Number"); + assertError(resolver, sql("Orders", name), true, "cannot be represented safely"); + } + + private static MetricFieldResolver resolver(String dataset, String field, String sourceType, String targetType) { + return new MetricFieldResolver(Map.of("datasets", List.of(dataset(dataset, field(field, sourceType)))), + Map.of("semanticDataObjects", List.of(targetDataset(dataset, targetField(field, targetType))))); + } + + private static MetricFieldResolver graphResolver(List> relationships, String... datasets) { + return new MetricFieldResolver(Map.of(), Map.of( + "semanticDataObjects", java.util.Arrays.stream(datasets).map(name -> targetDataset(name)).toList(), + "semanticRelationships", relationships)); + } + + private static Map relationship(String left, String right) { + return Map.of("leftSemanticDefinitionApiName", left, "rightSemanticDefinitionApiName", right); + } + + private static List sql(String... parts) { + return java.util.Arrays.stream(parts).map(part -> new Identifier(part, false)).toList(); + } + + @SafeVarargs + private static Map dataset(String name, Map... fields) { + return Map.of("name", name, "source", name + "__dll", "fields", List.of(fields)); + } + + private static Map field(String name, String datatype) { + return Map.of("name", name, "datatype", datatype); + } + + @SafeVarargs + private static Map targetDataset(String name, Map... fields) { + return Map.of("apiName", name, "semanticMeasurements", List.of(fields)); + } + + private static Map targetField(String name, String datatype) { + return Map.of("apiName", name, "dataType", datatype); + } + + private static void assertError(MetricFieldResolver resolver, List parts, + boolean tableau, String expected) { + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> resolver.resolve(parts, tableau)); + assertTrue(error.getMessage().contains(expected), error.getMessage()); + } +} From fff4101347f57511613fe59cd463acf736c001dc Mon Sep 17 00:00:00 2001 From: Saurabh Deshpande <43935865+saurabhdeshp@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:54:45 -0700 Subject: [PATCH 3/6] fix(salesforce): exclude disabled relationship paths --- converters/salesforce/README.md | 2 +- .../org/apache/ossie/converter/MetricFieldResolver.java | 3 +++ .../apache/ossie/converter/MetricFieldResolverTest.java | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md index 7130165f..864c5691 100644 --- a/converters/salesforce/README.md +++ b/converters/salesforce/README.md @@ -305,7 +305,7 @@ fractional expression. Missing metric types are inferred. A formula must be aggregated or constant: mixed row/aggregate expressions, nested aggregates, and aggregates without a dataset field fail. A single aggregate cannot combine fields from multiple datasets. Separate aggregates can use datasets connected through -exported relationships; connectivity alone does not prove join grain. +enabled exported relationships; connectivity alone does not prove join grain. **Limits and compatibility.** `COUNT`/`COUNTD` take a field. `ROUND` supports one argument or a second integer-literal precision; rounding-mode overloads are not diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java index 5da5dcc8..edda8a6e 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java @@ -65,6 +65,9 @@ void validateDatasets(Set referenced) { } } for (Map relationship : relationships) { + if (Boolean.FALSE.equals(relationship.get("isEnabled"))) { + continue; + } String left = getString(relationship, "leftSemanticDefinitionApiName"); String right = getString(relationship, "rightSemanticDefinitionApiName"); if (graph.containsKey(left) && graph.containsKey(right)) { diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java index fa6c4c8f..2f27d585 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java @@ -194,6 +194,15 @@ private static MetricFieldResolver resolver(String dataset, String field, String Map.of("semanticDataObjects", List.of(targetDataset(dataset, targetField(field, targetType))))); } + @Test + void disabledRelationshipsDoNotConnectDatasets() { + MetricFieldResolver resolver = graphResolver(List.of(Map.of( + "leftSemanticDefinitionApiName", "Orders", + "rightSemanticDefinitionApiName", "Returns", "isEnabled", false)), "Orders", "Returns"); + assertThrows(IllegalArgumentException.class, + () -> resolver.validateDatasets(java.util.Set.of("Orders", "Returns"))); + } + private static MetricFieldResolver graphResolver(List> relationships, String... datasets) { return new MetricFieldResolver(Map.of(), Map.of( "semanticDataObjects", java.util.Arrays.stream(datasets).map(name -> targetDataset(name)).toList(), From 9cda9e5176763e3a02dfd360fe3e7582d93112ff Mon Sep 17 00:00:00 2001 From: Saurabh Deshpande <43935865+saurabhdeshp@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:26:20 -0700 Subject: [PATCH 4/6] feat(salesforce): compile bound OSI expressions to Tua Use JSqlParser with a shared typed expression pipeline for derived fields and dependent metrics. Preserve semantic identities through environment bindings, validate relationships and final model references, and reject unsupported expressions or ambiguous metadata before export. Validation: 442 tests passed with no skips, Apache RAT passed, and packaged CLI export, import, and physical-binding smoke checks passed. --- converters/salesforce/README.md | 354 ++++++----- converters/salesforce/pom.xml | 15 + .../ossie/app/OssieSalesforceConverter.java | 24 +- .../ossie/converter/AbstractConverter.java | 6 + .../ossie/converter/ConversionContext.java | 39 ++ .../ossie/converter/ConverterFactory.java | 5 + .../apache/ossie/converter/ConverterImpl.java | 44 +- .../converter/CustomExtensionHandler.java | 49 +- .../ossie/converter/ExpressionAnalyzer.java | 227 +++++++ .../apache/ossie/converter/ExpressionAst.java | 58 ++ .../ossie/converter/ExpressionCompiler.java | 98 +++ .../converter/ExpressionFunctionRegistry.java | 76 +++ .../ossie/converter/ExpressionTokens.java | 112 ++++ .../ossie/converter/FieldExpressionPlan.java | 385 ++++++++++++ .../ossie/converter/FieldMappingHandler.java | 340 +++-------- .../converter/MetricCompilationPlan.java | 96 +++ .../converter/MetricExpressionTranslator.java | 551 +---------------- .../ossie/converter/MetricFieldResolver.java | 288 ++++----- .../ossie/converter/MetricMappingHandler.java | 33 +- .../converter/RelationshipMappingHandler.java | 122 +--- .../ossie/converter/SalesforceBindings.java | 174 ++++++ .../converter/SalesforceModelValidator.java | 567 ++++++++++++++++++ .../ossie/converter/SqlExpressionParser.java | 188 ++++++ .../ossie/converter/TuaExpressionEmitter.java | 83 +++ .../ossie/converter/TuaExpressionParser.java | 149 +++++ .../converter/pipeline/PipelineStep.java | 6 + .../java/org/apache/ossie/MetricCliTest.java | 155 +++++ .../ossie/MetricExportIntegrationTest.java | 124 +++- .../ossie/OssieToSalesforceConverterTest.java | 24 +- .../ossie/SalesforceToOssieConverterTest.java | 4 +- .../converter/ConstantFieldMetricTest.java | 148 +++++ .../converter/CustomExtensionHandlerTest.java | 120 ++++ .../converter/ExpressionCompilerTest.java | 194 ++++++ .../converter/FieldExpressionPlanTest.java | 346 +++++++++++ .../converter/MetricCompilationPlanTest.java | 133 ++++ .../MetricExpressionSemanticsTest.java | 127 +++- .../MetricExpressionTranslatorTest.java | 10 +- .../RelationshipMappingHandlerTest.java | 257 ++++++++ .../converter/SalesforceBindingsTest.java | 221 +++++++ .../SalesforceModelValidatorTest.java | 362 +++++++++++ .../SourceDocumentValidationTest.java | 92 +++ .../resources/examples/ossieToSalesforce.yaml | 29 - 42 files changed, 5151 insertions(+), 1284 deletions(-) create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java create mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java create mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md index 864c5691..af3880e7 100644 --- a/converters/salesforce/README.md +++ b/converters/salesforce/README.md @@ -22,8 +22,10 @@ A two-way converter between [Ossie semantic models](../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html). This converter supports conversion in both directions between Ossie YAML and -Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in -`custom_extensions`; see the mapping reference for direction-specific limits. +Salesforce Semantic Model JSON. Each input must contain one document; duplicate +mapping keys and trailing documents are rejected before conversion. Supported +unmapped Salesforce properties are preserved in `custom_extensions`; see the +mapping reference for direction-specific limits. ## Requirements @@ -42,7 +44,8 @@ This produces a self-contained executable jar at `target/ossie-salesforce-conver ## Setup -Obtain the Salesforce schema before building so it is bundled into the jar. +Both conversion directions validate input and output. Obtain the Salesforce schema +before building so it is bundled into the jar; conversion fails if it is missing. Maven copies the canonical Ossie schema from `../../core-spec/ossie-schema.json`. ### Salesforce Semantic Model Schema @@ -57,9 +60,10 @@ Run the complete suite, including Salesforce schema checks, with: mvn -DrequireSalesforceSchema=true clean verify ``` -The property makes a missing Salesforce schema fail the test run. Without it, -schema-dependent tests retain their existing skip behavior. `verify` also checks -Apache license headers. Do not commit downloaded schemas. +The property explicitly fails the suite when the Salesforce schema is missing, +including tests that otherwise skip for missing resources. Public API and CLI +checks require both schemas. `verify` also checks Apache license headers. Do not +commit downloaded schemas. ## Usage @@ -134,8 +138,8 @@ ossieToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); ### Features -- **Schema-validated** - Input is validated against JSON Schema before processing -- **Lossless conversion** - Unmapped properties are preserved in `custom_extensions` +- **Schema-validated** - Input and final output are validated against JSON Schema +- **Explicit conversion boundaries** - Supported native metadata is preserved; unsupported expressions, missing references and omitted declared entities fail conversion - **Bidirectional** - Supports both directions, with direction-specific limits documented below - **Supports Ossie Specification v0.2.0.dev0** @@ -168,7 +172,7 @@ ossieToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); | `datasets[].name` | `semanticDataObjects[].apiName` | | `datasets[].source` | `semanticDataObjects[].dataObjectName` | | Direct `fields[]` | Split into `semanticDimensions[]` and `semanticMeasurements[]` based on `dimension` presence | -| Calculated Tableau fields | `semanticCalculatedDimensions[]` through the existing expression-analysis path | +| Derived row fields | Validated Tua in `semanticCalculatedDimensions[]`, with dataset-qualified generated names | | `expression.dialects[].expression` | `dataObjectFieldName` | | Field `datatype` | Field `dataType` when a safe mapping exists | | `relationships[]` | `semanticRelationships[]` | @@ -207,22 +211,22 @@ type exists: | `Boolean` | `Boolean` | | `Date` | `Date` | | `DateTime`, `DateTimeTz` | `DateTime` | -| `Time`, `Opaque` | Omitted with a warning unless an exact extension type exists | +| `Time`, `Opaque` | Rejected unless a compatible exact native extension type exists | Salesforce has one `DateTime` type, so exporting timezone-free Ossie `DateTime` loses its distinction from `DateTimeTz`; the converter logs a warning because a subsequent Salesforce import interprets that value as `DateTimeTz`. An exact Salesforce extension value takes precedence over the portable mapping. -If it conflicts with `datatype`, the converter preserves the exact Salesforce -value and logs a warning. +If it conflicts with `datatype`, conversion fails. An absent direct-field datatype +can remain unspecified in metadata, but an expression using it needs a known, +compatible type. ### Field Role and Time Dimensions `datatype` does not determine whether an Ossie field is a dimension or a fact. For direct fields, the presence of the `dimension` object determines whether the -field is exported to `semanticDimensions` or `semanticMeasurements`. A calculated -Tableau expression follows the converter's existing calculated-dimension path. +field is exported to `semanticDimensions` or `semanticMeasurements`. A derived row expression becomes a model-level calculated dimension. On import, Salesforce `Date` and `DateTime` dimensions set `dimension.is_time` to `true`; other dimension types set it to `false`. On export, `dimension.is_time` @@ -230,38 +234,78 @@ does not invent or override a scalar type. This preserves Ossie's separation of logical data type from temporal role, including integer year and string month dimensions. -### Relationship Handling +### Relationships -**Unsupported relationships** (containing Formula or SemanticField types) are stored in `custom_extensions` at the model level rather than being converted to Ossie relationships. +Every declared relationship must be exported with the same endpoint and ordered +join-key pairs. Missing fields, unequal composite-key lengths, duplicate names, +and calculated join keys fail conversion; no relationship is silently filtered. +The default cardinality is `ManyToOne`, following OSI. Valid explicit Salesforce +cardinality metadata is preserved for native round trips. Declared primary and +unique keys are checked against references and the unique side of the chosen +cardinality. This checks metadata consistency, not uniqueness in actual data. + +On Salesforce import, unsupported Formula/SemanticField relationships remain in +model extensions. Export currently rejects those joins explicitly. Core +relationships and native extension arrays must not cause one another to disappear. +An extension-only relationship with omitted enablement metadata cannot establish +proven connectivity for a cross-dataset calculation. + +### Field expressions and physical bindings + +Field conversion parses the selected expression dialect (`TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`) and builds one dependency plan per model. Every declared field must either produce a direct field or a supported calculated dimension. Unsupported expressions fail with the dataset and field name; they are never silently omitted. + +A direct SQL identifier defines a physical column. For example, a field named `amount` with expression `revenue__c` emits `apiName: amount` and `dataObjectFieldName: revenue__c`. Double-quoted identifiers preserve their contents, including spaces, operators, periods, and escaped quotes. A verified qualification such as `warehouse.sales.orders.revenue__c` is reduced to its column name. Qualifiers must match the declared dataset name or its physical source; the converter does not guess unrelated catalog paths. + +A derived SQL expression can reference declared fields or physical columns exposed by direct fields in that same dataset. For example, with direct fields `amount = revenue__c` and `cost = cost__c`, both `amount - cost` and `revenue__c - cost__c` bind to `[Orders].[amount] - [Orders].[cost]`. A name matching different physical columns or semantic fields is rejected as ambiguous. SQL unquoted identifiers use case folding; quoted identifiers preserve case. A single-identifier SQL expression always defines a physical binding, even when it matches another semantic field name. A native `TABLEAU` expression such as `[Orders].[amount]` instead denotes a semantic alias and is emitted as a calculated dimension. + +Derived fields can depend on other derived fields, including declarations appearing later in the input. The plan expands those dependencies into row expressions over direct semantic fields. This lets metrics consume derived fields without relying on an undocumented global calculated-field reference syntax. Each calculated dimension uses `syntax: Tua`, a stable name based on `dataset__field`, and flattened direct-field dependencies. Name collisions, including collisions with metric names and sanitized names from other datasets, receive deterministic hash suffixes. + +Row fields must stay within their dataset and cannot contain aggregations. +A constant-only row field can be emitted as a calculated dimension, but a metric +cannot reference it until a native dataset anchor is supported; otherwise +`SUM(dataset.constant_one)` would collapse to an unscoped `SUM(1)`. Put aggregate expressions in `metrics`. Referenced fields need supported, compatible datatypes; inferred result types must agree with declared types. Unknown references, dependency cycles, unsupported operations, and incompatible types stop conversion. Dependency chains are limited to 128 fields, and the shared expression emitter limits expanded formulas to 131,072 characters. These are converter resource limits, not advertised Tableau platform limits. + +Environment-specific Salesforce bindings are applied after expressions have been bound in their original source scope. They can change the target data object and direct physical column names while preserving semantic API names, formulas, dependency identities, and the OSI input. Thus rebinding `amount` to `NetRevenue__c` still leaves its formulas referring to `[Orders].[amount]`. + +On Salesforce import, direct `dataObjectFieldName` values are represented as quoted `ANSI_SQL` identifiers. Calculated expressions remain `TABLEAU`. This preserves punctuation and avoids mislabeling physical column names as native Tableau formulas. + +These checks establish local binding and supported translation behavior. Native formula validation and result equivalence still require validation against the intended Tableau Next environment and dataset. ### Metric expressions -Metrics select `TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`, independent of entry -order. The selected expression is parsed and validated; an invalid preferred -expression fails rather than falling back to another dialect. Duplicate selected -dialect entries are errors. Successful conversion exports every declared metric. +Fields and metrics select `TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`, independent +of entry order. Duplicate dialect entries, an empty selected expression or an +unsupported selected expression fail without falling back to another dialect. +The source OSI model needs no new dialect entries or formula edits. The target is the Salesforce/Tableau Next semantic model's [Tua calculation language](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-functions.html). -Calculated measurements emit `syntax: Tua`, `dataType: Number`, and -`aggregationType: UserAgg`, so an already aggregated formula is not aggregated -again. See [calculated fields](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-calculated-fields.html) +Calculated measurements emit `syntax: Tua`, `aggregationType: UserAgg` and +`level: AggregateFunction`. The default numeric `dataType` is `Number`; compatible +native `Currency`/`Percentage` and display metadata such as labels and decimal +places are retained. Stale extension expressions cannot replace compiled formulas. Salesforce +extension data must be a single JSON object with unique keys; invalid metadata +fails with the owning entity name. +See [calculated fields](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-calculated-fields.html) and [aggregation rules](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-aggregation.html). -| SQL input | Tua output | -|-----------|------------| +| SQL input | Tua output / restriction | +|-----------|--------------------------| | `SUM`, `AVG`, `MIN`, `MAX`, `COUNT(field)` | Same aggregate | | `COUNT(DISTINCT field)` | `COUNTD(field)` | | `+`, `-`, `*`, `/`, parentheses, numeric constants | Explicitly grouped arithmetic | | Searched `CASE WHEN` | `IF … THEN … ELSEIF … ELSE … END` | | Comparisons, `AND`, `OR`, `NOT` | Equivalent grouped operators | -| `COALESCE(a, b, …)` | Nested `IFNULL` | +| `COALESCE(a, b, …)` | Nested two-argument `IFNULL` | | `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | | `IS NULL`, `IS NOT NULL` | `ISNULL`, `NOT ISNULL` | | `ABS`, `ROUND`, `CEIL`, `FLOOR` | `ABS`, `ROUND`, `CEILING`, `FLOOR` | +| `YEAR(date)` | `YEAR`; requires `Date` or timezone-free `DateTime` | +| `LENGTH(text)` | `LEN` | +| `POSITION(needle IN text)` | `FIND(text, needle)` | +| `SUBSTRING(text, start[, length])` | `MID`; start must be provably positive and optional length nonnegative | -These constructs compose. For example, with declared numeric fields `profit` and -`revenue` in dataset `orders`: +These constructs compose. For example: ```yaml metrics: @@ -270,136 +314,170 @@ metrics: expression: dialects: - dialect: SNOWFLAKE - expression: SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0) + expression: total_profit / NULLIF(total_revenue, 0) + - name: total_profit + datatype: Decimal + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(orders.profit) + - name: total_revenue + datatype: Decimal + expression: + dialects: + - dialect: SNOWFLAKE + expression: SUM(orders.revenue) ``` -The resulting expression is: - -```text -(SUM([orders].[profit]) / (IF (SUM([orders].[revenue]) = 0) THEN NULL ELSE SUM([orders].[revenue]) END)) -``` +Named metric references resolve independently of declaration order, are checked +for cycles and are inlined as aggregate expressions. Compiled dependencies are +cached per model. `[metric]` is the corresponding native `TABLEAU` reference. +A field and metric sharing an unqualified name are ambiguous; qualify the field +or use a unique metric name. Aggregating an already aggregated metric fails. -**Binding and types.** References resolve against declared dataset and field -names, then against fields actually emitted to Salesforce. Physical column names -and source paths are not aliases. Unqualified SQL fields must be unique. Regular -SQL names normalize to uppercase; double-quoted names match the normalized +**Binding and types.** Metric references use declared logical field names, +including supported derived fields. Physical column names and source paths are +not metric aliases. Unqualified SQL fields must be unique across datasets. +Regular SQL names normalize to uppercase; double-quoted names match the normalized declaration exactly, following the [expression specification](../../core-spec/expression_language.md). Thus `"ORDERS"."AMOUNT"` matches regular declarations `orders.amount`, while -`"orders"."amount"` requires explicitly quoted lowercase declarations. Target -API names are preserved; conversion does not rename fields or discover columns. +`"orders"."amount"` requires explicitly quoted lowercase declarations. Native +`TABLEAU` uses exact `[dataset].[field]` names. Legacy complete bracket references +in `ANSI_SQL` are retained as a compatibility case; Snowflake requires SQL quotes. Names containing brackets or control characters fail because their Tua escaping is not established. -`TABLEAU` references use exact `[dataset].[field]` API names. Its supported formula -subset is the Tua equivalents above, including `IF`, `IFNULL`, `ISNULL`, `COUNTD` -and `CEILING`. Existing `ANSI_SQL` expressions using complete bracket notation -retain that spelling as a compatibility case and receive the same validation. -Bracket notation is not accepted as Snowflake SQL. - -Fields need a known compatible datatype, either declared in OSI or restored from -an existing Salesforce field type. Arithmetic and `SUM`/`AVG` require numbers; -`MIN`/`MAX` also permit text and temporal values inside numeric calculations. -Comparisons and conditional/null-handling branches must have compatible types. -Metrics must return numbers; a declared `Integer` result cannot conceal a -fractional expression. Missing metric types are inferred. A formula must be -aggregated or constant: mixed row/aggregate expressions, nested aggregates, and -aggregates without a dataset field fail. A single aggregate cannot combine fields -from multiple datasets. Separate aggregates can use datasets connected through -enabled exported relationships; connectivity alone does not prove join grain. - -**Limits and compatibility.** `COUNT`/`COUNTD` take a field. `ROUND` supports one -argument or a second integer-literal precision; rounding-mode overloads are not -supported. `CEIL`/`FLOOR` take one argument. Simple `CASE`, date/time and string -functions, casts, metric/calculated-field references, windows, LOD, `COUNT(*)`, -SQL comments and backslash string escapes are outside this subset. String and -Boolean literals are supported in predicates. No null-to-zero setting or implicit -cast is added. Errors name the metric and explain the rejected construct or -reference. Literal zero divisors fail; use `NULLIF` to make a zero denominator -nullable. Expressions have bounded size, nesting and generated output. - -This replaces the unvalidated SQL fallback introduced in -[#402](https://github.com/apache/ossie/pull/402). Previously accepted invalid, -unsupported or untyped formulas now fail, including invalid `TABLEAU` input. -Metric/model extension preservation remains owned by #402. CLI conversion errors -are printed to stderr with exit code 3; this small change overlaps the conversion -error handling in [#286](https://github.com/apache/ossie/pull/286). - -**Implementation choice.** The converter reuses its mapping pipeline, datatype -mapper and exceptions. Its bounded recursive-descent parser synthesizes a typed -Tua formula and aggregation level at each production. Add expression support in -the relevant parser/function rule with type, aggregation and composition tests. -It adds no dependencies. The open [#222](https://github.com/apache/ossie/pull/222) -implements an Ossie SQLGlot dialect in Python; it has no Tua emitter or model-bound -field/type checks. Using it here would require a Python runtime bridge and the -same target checks. The Java implementation follows the specification's naming -rules and SQL `NOT` precedence, including #222's proposed precedence correction, -without implementing a new shared expression framework. - -**Validation boundary.** Unit tests cover parsing, binding and failure behavior. -Independent local Tua evaluation tests use synthetic rows with nulls, duplicates, -empty inputs and zero denominators. Those tests model the intended semantics; -they are not native Tableau Next execution. The published Salesforce **output** -schema checks structure, not formula syntax, catalog bindings or authoring API -acceptance. Before deployment, validate authoring and native queries in a test -org, including numeric precision, rounding ties, empty groups, null behavior and -unguarded dynamic division and multi-dataset grain. This converter does not provision Data 360 bindings or enrich -missing fields. +Referenced fields need known compatible datatypes. Arithmetic and `SUM`/`AVG` +require numbers; branches and comparisons require compatible operand types. +Metrics must return numbers and be aggregated or constant. Declared `Integer` +results cannot conceal fractional expressions. `CEIL`, `FLOOR` and `ROUND` at +nonpositive precision infer integral values. All-null metrics need an explicit +numeric datatype. Mixed row/aggregate expressions and nested aggregates fail. + +One aggregate cannot combine fields from different datasets. Separate aggregates +may reference datasets connected through enabled exported relationships, including +references introduced by metric dependencies. Connectivity checks do not prove +that the target query planner preserves the intended grain. + +**Boundaries.** Parser acceptance never implies target support. Windows, LOD, +subqueries, SQL casts, simple `CASE`, UDFs, `COUNT(*)`, comments, backslash string +escapes, implicit type coercions and unlisted functions are unsupported. No raw +SQL fallback is emitted. `COUNT`/`COUNTD` accept declared fields. `ROUND` accepts +one argument or an integer-literal precision; rounding-mode overloads are rejected. +Literal zero divisors fail; `NULLIF` can guard a denominator. Each expression is +bounded to 32,768 input characters, 8,192 tokens, 128 syntax/dependency levels and +131,072 generated characters. These are converter resource limits, not platform +limits. Errors identify the field/metric and rejected construct or reference. + +### Environment bindings + +Use an optional JSON or YAML manifest to bind the same OSI model to a Salesforce +environment. This file contains deployment names, never business formulas: -## Architecture +```yaml +models: + sales: + dataspace: default + datasets: + orders: + dataObjectName: Orders__dll + dataObjectType: Dlo + fields: + profit: NetProfit__c + revenue: Revenue__c +``` +Only listed values are overridden. Model, dataset and field keys are exact OSI +names. Fields must be direct physical bindings. Unknown names/properties, +duplicate keys, multiple YAML documents and bindings for calculated fields fail. +Native object types must match the bundled Salesforce schema; this is not catalog +discovery or DLO/DMO provisioning. Semantic names, formulas and the OSI file remain +unchanged. Without a manifest, existing source and extension mappings apply. + +```bash +java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar \ + toSF input.yaml --bindings bindings.yaml ``` - ┌───────────────────────┐ - │ OssieSalesforceConverter│ - │ (CLI App) │ - └───────────┬───────────┘ - │ - ┌───────┴────────┐ - │ ConverterFactory│ - └───────┬────────┘ - │ - ┌─────────────┴─────────────┐ - │ ConverterImpl │ - │ (Pipeline-based) │ - │ │ - │ • Configurable pipeline │ - │ • Bidirectional mapping │ - └─────────────┬─────────────┘ - │ - ┌─────────────┴─────────────┐ - │ Pipeline Handlers │ - ├───────────────────────────┤ - │ • DatasetMappingHandler │ - │ • FieldMappingHandler │ - │ • RelationshipHandler │ - │ • MetricMappingHandler │ - │ • SemanticModelHandler │ - └─────────────┬─────────────┘ - │ - ┌─────────────┴─────────────┐ - │ Support Components │ - ├───────────────────────────┤ - │ • GenericMappingEngine │ - │ • CustomExtensionHandler │ - │ • SchemaValidator │ - └───────────────────────────┘ + +```java +SalesforceBindings bindings = SalesforceBindings.fromPath(Path.of("bindings.yaml")); +Converter converter = ConverterFactory.getConverter( + ConversionDirection.OSSIE_TO_SALESFORCE, bindings); +List output = converter.convert(osiYaml); ``` -**ConverterFactory** — Creates converter instances for specified direction +Bindings are export-only. CLI usage/input errors exit with code 1/2; conversion +and schema errors are printed to stderr with exit code 3. All models are converted +and validated before the file API begins writing any output, so conversion failure +in a later model does not leave earlier model files behind. -**Pipeline Configuration** — Handlers and direction-specific settings defined in `ossie-salesforce-converter-config.yaml` +## Architecture -**GenericMappingEngine** — Path-based property mapping using `mappings.yaml` configuration +```text +OSI input -> source schema validation -> ConversionContext (one per model) + -> dataset mapping + -> FieldExpressionPlan: classify physical fields, bind and compile derived fields + -> validated relationships + -> MetricCompilationPlan: resolve fields/metrics, detect cycles, compile dependencies + -> native metadata restoration + -> physical environment bindings + -> final identity/reference/coverage checks -> target schema validation -> output + +ExpressionCompiler: + SQL -> JSqlParser frontend --+ + +-> immutable AST -> typed/aggregation analysis -> Tua emitter + TABLEAU -> bounded frontend -+ +``` + +JSqlParser 5.3 is used under its Apache-2.0 option for SQL syntax parsing. It does +not provide Tua semantics. `ExpressionFunctionRegistry` defines supported +dialect/function/arity combinations; `ExpressionAnalyzer` checks types and +aggregation; `TuaExpressionEmitter` writes only validated nodes. Supporting a new +function requires an explicit registry entry, semantic checks, lowering rule and +tests. SQLGlot is not required at runtime, and no Python bridge is introduced. + +`MetricFieldResolver` builds model-scoped identity and relationship indexes once. +Field and metric dependency plans cache compilation while enforcing depth and +output limits. No model state is stored in reusable handlers. The existing generic +mapping pipeline, schema resources, datatype mapper and public converter APIs remain +in use. `PipelineStep` retains the original map-based method for custom handlers. + +Final validation detects dangling references, duplicate identities, omitted core +or native-extension entities, disconnected calculations and inconsistent join keys. +Extension-only calculations also pass the shared expression compiler. Schema +validation runs after all extensions and bindings; neither can bypass the final +checks. A native extension array that conflicts with an emitted core array fails +instead of silently losing distinct entities. -**CustomExtensionHandler** — Preserves unmapped Salesforce properties in Ossie's `custom_extensions` for lossless bi-directional conversion +## Validation + +```bash +mvn -DrequireSalesforceSchema=true clean verify +``` -**SchemaValidator** — Validates input against JSON schemas before conversion +The suite covers parser/AST rejection, quoted names, types, aggregation, field and +metric dependency graphs, binding overlays, relationship preservation, native +metadata, CLI failures and both conversion directions. An independent local Tua +evaluator checks results over synthetic rows including nulls, duplicates, empty +inputs, decimals, dates and string fields. Stress cases exercise dependency depth, +expansion bounds and many metrics sharing one dependency. `verify` also runs Apache +RAT license-header checks. + +These are local checks, not native Tableau Next execution. The Salesforce output +schema validates structure, not tenant catalog existence or backend formula +semantics. Deployment still needs native formula/authoring validation and result +comparison in the intended org, especially numeric precision, rounding ties, nulls, +empty groups, timezones and multi-dataset grain. General Snowflake/ANSI SQL cannot +be promised equivalent where Tua has no supported construct. Unsupported cases +must use an explicit new lowering or a separately designed native execution route. ## Examples -See the test suite for sample models demonstrating various features: -- `src/test/resources/examples/ossieToSalesforce.yaml` - Ossie model example -- `src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java` - Ossie to Salesforce conversion tests -- `src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java` - Salesforce to Ossie conversion tests +- `src/test/resources/examples/ossieToSalesforce.yaml`: valid OSI export fixture +- `src/test/java/org/apache/ossie/MetricExportIntegrationTest.java`: public API, dependencies and bindings +- `src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java`: supported/rejected expressions +- `src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java`: derived fields and scope +- `src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java`: reverse conversion ## License diff --git a/converters/salesforce/pom.xml b/converters/salesforce/pom.xml index 396c9bdc..ff3a4f83 100644 --- a/converters/salesforce/pom.xml +++ b/converters/salesforce/pom.xml @@ -52,9 +52,24 @@ 1.5.9 3.6.2 3.5.0 + 5.3 + + + com.github.jsqlparser + jsqlparser + ${jsqlparser.version} + + + + org.openjdk.jmh + jmh-core + + + com.fasterxml.jackson.core jackson-databind diff --git a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java index c6e59090..35c01fc5 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java @@ -22,8 +22,10 @@ import org.apache.ossie.converter.Converter; import org.apache.ossie.converter.ConverterFactory; import org.apache.ossie.converter.ConversionDirection; +import org.apache.ossie.converter.SalesforceBindings; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.exception.InvalidInputException; +import org.apache.ossie.exception.ValidationException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -39,6 +41,7 @@ public class OssieSalesforceConverter { public static void main(String[] args) { if (args.length < 2) { + System.err.println("Usage: toSF [--bindings ] | toOssie "); System.exit(1); } @@ -48,17 +51,26 @@ public static void main(String[] args) { Path inputPath = Paths.get(args[1]); ConversionDirection direction = parseDirection(directionArg); - app.convert(direction, inputPath); + SalesforceBindings bindings = SalesforceBindings.none(); + if (args.length != 2) { + if (args.length != 4 || !"--bindings".equals(args[2]) + || direction != ConversionDirection.OSSIE_TO_SALESFORCE) { + throw new InvalidInputException("Expected toSF [--bindings ]"); + } + bindings = SalesforceBindings.fromPath(Paths.get(args[3])); + } + app.convert(direction, inputPath, bindings); } catch (InvalidInputException e) { + System.err.println("Error: " + e.getMessage()); System.exit(2); - } catch (ConversionException e) { + } catch (ConversionException | ValidationException e) { System.err.println("Error: " + e.getMessage()); System.exit(3); } } private static ConversionDirection parseDirection(String direction) { - return switch (direction.toLowerCase()) { + return switch (direction.toLowerCase(java.util.Locale.ROOT)) { case "tosf" -> ConversionDirection.OSSIE_TO_SALESFORCE; case "toossie" -> ConversionDirection.SALESFORCE_TO_OSSIE; default -> throw new InvalidInputException( @@ -75,6 +87,10 @@ private static ConversionDirection parseDirection(String direction) { * @param inputPath path to the input file */ public void convert(ConversionDirection direction, Path inputPath) { + convert(direction, inputPath, SalesforceBindings.none()); + } + + public void convert(ConversionDirection direction, Path inputPath, SalesforceBindings bindings) { if (!Files.exists(inputPath)) { throw new InvalidInputException("Input file not found: " + inputPath); } @@ -84,7 +100,7 @@ public void convert(ConversionDirection direction, Path inputPath) { outputDir = Path.of("."); } - Converter converter = ConverterFactory.getConverter(direction); + Converter converter = ConverterFactory.getConverter(direction, bindings); converter.convert(inputPath, outputDir); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java index 88a51804..77f51b6f 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java @@ -20,9 +20,11 @@ package org.apache.ossie.converter; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import org.apache.ossie.exception.ConversionException; @@ -78,14 +80,18 @@ protected AbstractConverter(PropertyMapper mapper) { this.mapper = mapper; this.jsonMapper = new ObjectMapper() + .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) .enable(SerializationFeature.INDENT_OUTPUT); YAMLFactory yamlFactory = new YAMLFactory() .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); + yamlFactory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); this.yamlMapper = new ObjectMapper(yamlFactory) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) .enable(SerializationFeature.INDENT_OUTPUT); this.customExtensionHandler = new CustomExtensionHandler(this.jsonMapper); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java new file mode 100644 index 00000000..bc9859ed --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import java.util.Map; + +/** State owned by one model conversion; never shared across inputs or converter calls. */ +public final class ConversionContext { + private final Map sourceData; + private final Map outputData; + private FieldExpressionPlan fieldPlan; + + public ConversionContext(Map sourceData, Map outputData) { + this.sourceData = sourceData; + this.outputData = outputData; + } + + public Map sourceData() { return sourceData; } + public Map outputData() { return outputData; } + FieldExpressionPlan fieldPlan() { return fieldPlan; } + void fieldPlan(FieldExpressionPlan plan) { this.fieldPlan = plan; } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java index 657e9504..2a7b0c4e 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java @@ -34,4 +34,9 @@ public class ConverterFactory { public static Converter getConverter(ConversionDirection direction) { return new ConverterImpl(direction); } + + /** Creates a converter with an external environment binding catalog. */ + public static Converter getConverter(ConversionDirection direction, SalesforceBindings bindings) { + return new ConverterImpl(direction, bindings); + } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java index c3735a4d..f942f5f7 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ossie.converter.pipeline.*; -import org.apache.ossie.converter.pipeline.*; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; @@ -45,14 +44,28 @@ public class ConverterImpl extends AbstractConverter { private final DirectionConfig directionConfig; private final List steps; private final SchemaValidator schemaValidator; + private final SchemaValidator targetSchemaValidator; + private final SalesforceBindings bindings; public ConverterImpl(ConversionDirection direction) { - this(direction, PipelineConfigLoader.loadFromResource()); + this(direction, PipelineConfigLoader.loadFromResource(), SalesforceBindings.none()); + } + + public ConverterImpl(ConversionDirection direction, SalesforceBindings bindings) { + this(direction, PipelineConfigLoader.loadFromResource(), bindings); } ConverterImpl(ConversionDirection direction, PipelineConfig config) { + this(direction, config, SalesforceBindings.none()); + } + + private ConverterImpl(ConversionDirection direction, PipelineConfig config, SalesforceBindings bindings) { super(); this.direction = direction; + this.bindings = java.util.Objects.requireNonNull(bindings, "bindings"); + if (direction != ConversionDirection.OSSIE_TO_SALESFORCE && !bindings.isEmpty()) { + throw new ConversionException("Salesforce bindings apply only to toSF conversion"); + } // Get handler list for this direction List handlerNames = config.getPipelines().get(direction.toPipelineKey()); @@ -74,6 +87,11 @@ public ConverterImpl(ConversionDirection direction) { directionConfig.getSchemaPath() ); + // Output validation is part of conversion, not only an optional test assertion. + this.targetSchemaValidator = new SchemaValidator(jsonMapper, + direction == ConversionDirection.OSSIE_TO_SALESFORCE + ? SchemaValidator.SALESFORCE_SCHEMA_PATH : SchemaValidator.OSSIE_SCHEMA_PATH); + // Initialize pipeline steps using factory HandlerFactory factory = new HandlerFactory(customExtensionHandler); this.steps = handlerNames.stream() @@ -99,6 +117,12 @@ public List convert(String content) { private List convertOssieToSalesforce(Map ossieRoot) { List semanticModels = getList(ossieRoot, SEMANTIC_MODEL); List results = new ArrayList<>(); + java.util.Set names = new java.util.HashSet<>(); + for (Object modelObj : semanticModels) { + String name = getString(asMap(modelObj), NAME); + if (!names.add(name)) throw new ConversionException("Duplicate model name '" + name + "'"); + } + bindings.validateModels(names); for (Object modelObj : semanticModels) { Map sourceData = asMap(modelObj); @@ -117,6 +141,7 @@ private List convertSalesforceToOssie(Map sourceData) { Map ossieRoot = new LinkedHashMap<>(); ossieRoot.put(VERSION, OSSIE_VERSION); ossieRoot.put(SEMANTIC_MODEL, List.of(outputData)); + targetSchemaValidator.validate(ossieRoot); return List.of(toYaml(ossieRoot)); } catch (JsonProcessingException e) { throw new ConversionException("Failed to wrap output in Ossie root", e); @@ -129,10 +154,21 @@ private String executePipeline(Map sourceData) { ? mapper.getOssieToSalesforceMappings() : mapper.getSalesforceToOssieMappings()); + ConversionContext context = new ConversionContext(sourceData, outputData); for (PipelineStep step : steps) { - step.execute(sourceData, outputData, mappings); + try { + step.execute(context, mappings); + } catch (IllegalArgumentException e) { + String name = getString(sourceData, + direction == ConversionDirection.OSSIE_TO_SALESFORCE ? NAME : API_NAME); + throw new ConversionException("Model '" + name + "': " + e.getMessage(), e); + } + } + if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { + bindings.apply(sourceData, outputData); + new SalesforceModelValidator().validate(sourceData, outputData, context.fieldPlan()); + targetSchemaValidator.validate(outputData); } - return serialize(outputData); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java index 849f6392..81331838 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java @@ -22,9 +22,13 @@ import static org.apache.ossie.converter.ConverterConstants.*; import static org.apache.ossie.util.DataStructureUtils.*; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ossie.converter.ConverterConstants.Level; +import org.apache.ossie.exception.ConversionException; import org.apache.ossie.util.PathUtils; import java.util.*; import org.slf4j.Logger; @@ -319,31 +323,36 @@ public void restoreSalesforceCustomExtension(Map sfItem, Map" : itemName) + "'"; + if (!(dataObj instanceof String dataJson)) { + throw new ConversionException(scope + " must contain a JSON object encoded as a string"); } + Map salesforceProperties; try { - // Parse JSON string to Map - Map salesforceProperties = jsonMapper.readValue( - (String) dataObj, - new TypeReference>() {} - ); - - String itemName = getString(ossieItem, NAME); - if (itemName == null) { - logger.warn("Item has no name, skipping custom_extensions restoration"); - return; - } + // Use a strict reader without changing the shared mapper or the reverse conversion path. + salesforceProperties = jsonMapper.readerFor(new TypeReference>() {}) + .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .with(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) + .readValue(dataJson); + } catch (JsonProcessingException e) { + throw new ConversionException(scope + " must contain one JSON object with unique property names: " + + e.getOriginalMessage(), e); + } + if (salesforceProperties == null) { + throw new ConversionException(scope + " must contain a JSON object"); + } - for (Map.Entry entry : salesforceProperties.entrySet()) { - if (!sfItem.containsKey(entry.getKey())) { - sfItem.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); - } - } + if (itemName == null) { + logger.warn("Item has no name, skipping custom_extensions restoration"); + return; + } - } catch (Exception e) { - logger.warn("Failed to restore custom_extensions: {}", e.getMessage()); + for (Map.Entry entry : salesforceProperties.entrySet()) { + if (!sfItem.containsKey(entry.getKey())) { + sfItem.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + } } }); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java new file mode 100644 index 00000000..2e4d274c --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.converter.ExpressionAst.*; +import static org.apache.ossie.converter.ExpressionCompiler.Level; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Resolves names and checks types, aggregation level and function domains before target rendering. */ +final class ExpressionAnalyzer { + private final String dialect; + private final ExpressionCompiler.ReferenceResolver resolver; + private int depth; + ExpressionAnalyzer(String dialect, ExpressionCompiler.ReferenceResolver resolver) { + this.dialect = dialect; this.resolver = resolver; + } + Typed analyze(Node node) { + if (++depth > 128) throw new IllegalArgumentException("expression nesting exceeds 128 levels"); + try { return analyzeNode(node); } finally { depth--; } + } + private Typed analyzeNode(Node node) { + if (node instanceof Literal literal) { + Object value = literal.value(); + Type type = value == null ? Type.NULL : value instanceof Boolean ? Type.BOOLEAN + : value instanceof String ? Type.STRING + : ((BigDecimal) value).stripTrailingZeros().scale() <= 0 ? Type.INTEGER : Type.DECIMAL; + return new Typed(node, type, Level.CONSTANT, Set.of(), List.of(), null, + value instanceof BigDecimal number ? number : null); + } + if (node instanceof Field field) { + ExpressionCompiler.Binding binding = resolver.resolve(field.reference()); + if (binding == null || binding.expression() == null || binding.expression().isBlank()) { + throw new IllegalArgumentException("field resolver returned no binding"); + } + Type type = Type.of(binding.datatype()); + if (type == Type.UNKNOWN) throw new IllegalArgumentException("field reference needs known field datatypes"); + return new Typed(node, type, binding.level(), binding.datasets(), List.of(), binding, null); + } + if (node instanceof Unary unary) { + Typed child = analyze(unary.operand()); + String operator = unary.operator(); + Type result = child.type(); + BigDecimal number = child.number(); + if (operator.equals("ISNULL")) { result = Type.BOOLEAN; number = null; } + else if (operator.equals("NOT")) { require(child, Type.BOOLEAN, "NOT"); result = Type.BOOLEAN; number = null; } + else { numeric(child, "unary " + operator); if (operator.equals("-") && number != null) number = number.negate(); } + return new Typed(node, result, child.level(), child.datasets(), List.of(child), null, number); + } + if (node instanceof Binary binary) { + Typed left = analyze(binary.left()); Typed right = analyze(binary.right()); + String op = binary.operator(); + Type result; + if (op.equals("AND") || op.equals("OR")) { + require(left, Type.BOOLEAN, op); require(right, Type.BOOLEAN, op); result = Type.BOOLEAN; + } else if (Set.of("=", "!=", "<", "<=", ">", ">=").contains(op)) { + compatible(left.type(), right.type(), "comparison"); + if (!Set.of("=", "!=").contains(op) && (left.type() == Type.BOOLEAN || right.type() == Type.BOOLEAN)) { + throw new IllegalArgumentException("ordered comparison requires numeric, text or temporal operands"); + } + result = Type.BOOLEAN; + } else { + numeric(left, op); numeric(right, op); + result = compatible(left.type(), right.type(), op); + if (op.equals("/")) { + if (right.number() != null && right.number().signum() == 0) { + throw new IllegalArgumentException("division by literal zero; use NULLIF(denominator, 0) for a nullable denominator"); + } + result = Type.DECIMAL; + } + } + return compose(node, result, List.of(left, right)); + } + if (node instanceof Conditional conditional) { + List children = new ArrayList<>(); + Type result = Type.NULL; + for (int i = 0; i < conditional.branches().size(); i += 2) { + Typed predicate = analyze(conditional.branches().get(i)); + require(predicate, Type.BOOLEAN, "conditional predicate"); + Typed branch = analyze(conditional.branches().get(i + 1)); + result = compatible(result, branch.type(), "conditional branches"); + children.add(predicate); children.add(branch); + } + Typed otherwise = analyze(conditional.otherwise()); + result = compatible(result, otherwise.type(), "conditional branches"); + children.add(otherwise); + return compose(node, result, children); + } + Call call = (Call) node; + ExpressionFunctionRegistry.Spec spec = ExpressionFunctionRegistry.require( + call.name(), dialect, call.arguments().size(), call.distinct()); + List arguments = call.arguments().stream().map(this::analyze).toList(); + return switch (spec.rule()) { + case AGGREGATE -> aggregate(call, arguments); + case COALESCE -> { + Type result = Type.NULL; + for (Typed argument : arguments) result = compatible(result, argument.type(), call.name() + " arguments"); + yield compose(node, result, arguments); + } + case NULLIF -> { + compatible(arguments.get(0).type(), arguments.get(1).type(), "NULLIF arguments"); + yield compose(node, arguments.get(0).type(), arguments); + } + case ISNULL -> compose(node, Type.BOOLEAN, arguments); + case NUMERIC -> numericFunction(call, arguments); + case YEAR -> { + Type input = arguments.get(0).type(); + if (!Set.of(Type.DATE, Type.DATETIME, Type.NULL).contains(input)) { + throw new IllegalArgumentException("YEAR requires Date or DateTime; timezone-dependent extraction is unsupported"); + } + yield compose(node, Type.INTEGER, arguments); + } + case LENGTH -> { + require(arguments.get(0), Type.STRING, call.name()); + yield compose(node, Type.INTEGER, arguments); + } + case POSITION -> { + for (Typed argument : arguments) require(argument, Type.STRING, call.name()); + yield compose(node, Type.INTEGER, arguments); + } + case SUBSTRING -> substring(call, arguments); + }; + } + private Typed aggregate(Call call, List arguments) { + String name = call.name(); Typed argument = arguments.get(0); + if (argument.level() == Level.AGGREGATE) throw new IllegalArgumentException("nested aggregate " + name + " is unsupported"); + if (argument.datasets().isEmpty()) throw new IllegalArgumentException(name + " needs a declared field to establish its dataset"); + if (argument.datasets().size() > 1) throw new IllegalArgumentException("one aggregate cannot combine fields from multiple datasets"); + boolean count = name.equals("COUNT") || name.equals("COUNTD"); + if (count && !(argument.node() instanceof Field)) { + throw new IllegalArgumentException(name + " requires a declared field; counting expressions is unsupported"); + } + if (name.equals("MIN") || name.equals("MAX")) { + if (argument.type() == Type.BOOLEAN || argument.type() == Type.UNKNOWN) { + throw new IllegalArgumentException(name + " requires numeric, text or temporal operands"); + } + } else if (!count) numeric(argument, name); + Type result = count ? Type.INTEGER : name.equals("AVG") ? Type.DECIMAL : argument.type(); + return new Typed(call, result, Level.AGGREGATE, argument.datasets(), arguments, null, null); + } + private Typed numericFunction(Call call, List arguments) { + Typed value = arguments.get(0); numeric(value, call.name()); + BigDecimal places = BigDecimal.ZERO; + if (arguments.size() == 2) { + places = arguments.get(1).number(); + if (places == null || places.stripTrailingZeros().scale() > 0 + || places.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0 + || places.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException("ROUND precision must be a 32-bit integer literal"); + } + } + Type result = value.type(); + if (result != Type.NULL && (Set.of("CEIL", "CEILING", "FLOOR").contains(call.name()) + || call.name().equals("ROUND") && places.signum() <= 0)) result = Type.INTEGER; + return compose(call, result, arguments); + } + private Typed substring(Call call, List arguments) { + require(arguments.get(0), Type.STRING, call.name()); + for (int i = 1; i < arguments.size(); i++) require(arguments.get(i), Type.INTEGER, call.name()); + // Snowflake allows non-positive indices with semantics different from Tua MID. + // Only proven common domains are lowered, including POSITION(...) + 1 in email-domain fields. + if (call.name().equals("SUBSTRING")) { + BigDecimal start = lowerBound(arguments.get(1)); + if (start == null || start.signum() <= 0) throw new IllegalArgumentException("SUBSTRING start must be provably positive for Tua MID"); + if (arguments.size() == 3) { + BigDecimal length = lowerBound(arguments.get(2)); + if (length == null || length.signum() < 0) throw new IllegalArgumentException("SUBSTRING length must be provably non-negative for Tua MID"); + } + } + return compose(call, Type.STRING, arguments); + } + private BigDecimal lowerBound(Typed value) { + if (value.number() != null) return value.number(); + if (value.node() instanceof Call call && Set.of("LENGTH", "LEN", "POSITION", "FIND").contains(call.name())) return BigDecimal.ZERO; + if (value.node() instanceof Binary binary && binary.operator().equals("+")) { + BigDecimal left = lowerBound(value.children().get(0)), right = lowerBound(value.children().get(1)); + return left == null || right == null ? null : left.add(right); + } + return null; + } + private Typed compose(Node node, Type type, List arguments) { + Level level = Level.CONSTANT; Set datasets = new HashSet<>(); + for (Typed argument : arguments) { + if (level != Level.CONSTANT && argument.level() != Level.CONSTANT && level != argument.level()) { + throw new IllegalArgumentException("cannot mix aggregate and unaggregated field expressions"); + } + if (argument.level() != Level.CONSTANT) level = argument.level(); + datasets.addAll(argument.datasets()); + } + return new Typed(node, type, level, datasets, arguments, null, null); + } + private static void numeric(Typed value, String context) { + if (!value.type().numeric() && value.type() != Type.NULL) { + throw new IllegalArgumentException(context + " requires numeric operands, found " + value.type() + "; declare a compatible field datatype"); + } + } + private static void require(Typed value, Type expected, String context) { + if (value.type() != expected && value.type() != Type.NULL) throw new IllegalArgumentException(context + " requires " + expected + ", found " + value.type()); + } + private static Type compatible(Type left, Type right, String context) { + if (left == Type.UNKNOWN || right == Type.UNKNOWN) throw new IllegalArgumentException(context + " needs known field datatypes"); + if (left == Type.NULL) return right; + if (right == Type.NULL || left == right) return left; + if (left.numeric() && right.numeric()) return left == Type.FLOAT || right == Type.FLOAT ? Type.FLOAT : Type.DECIMAL; + throw new IllegalArgumentException(context + " has incompatible types " + left + " and " + right); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java new file mode 100644 index 00000000..7091029d --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Set; + +/** Immutable compiler nodes. Neither source parsing nor type checking emits target text. */ +final class ExpressionAst { + private ExpressionAst() {} + sealed interface Node permits Literal, Field, Unary, Binary, Call, Conditional {} + record Literal(Object value) implements Node {} + record Field(ExpressionCompiler.Reference reference) implements Node {} + record Unary(String operator, Node operand) implements Node {} + record Binary(String operator, Node left, Node right) implements Node {} + record Call(String name, List arguments, boolean distinct) implements Node { + Call { arguments = List.copyOf(arguments); } + } + /** Alternating predicate/result pairs, followed by a separate ELSE expression. */ + record Conditional(List branches, Node otherwise) implements Node { + Conditional { branches = List.copyOf(branches); } + } + enum Type { + INTEGER("Integer"), DECIMAL("Decimal"), FLOAT("Float"), STRING("String"), + BOOLEAN("Boolean"), DATE("Date"), DATETIME("DateTime"), DATETIME_TZ("DateTimeTz"), + NULL(null), UNKNOWN(null); + final String datatype; + Type(String datatype) { this.datatype = datatype; } + boolean numeric() { return this == INTEGER || this == DECIMAL || this == FLOAT; } + static Type of(String datatype) { + if (datatype == null) return UNKNOWN; + for (Type type : values()) if (datatype.equals(type.datatype)) return type; + return UNKNOWN; + } + } + record Typed(Node node, Type type, ExpressionCompiler.Level level, Set datasets, + List children, ExpressionCompiler.Binding binding, BigDecimal number) { + Typed { datasets = Set.copyOf(datasets); children = List.copyOf(children); } + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java new file mode 100644 index 00000000..45af5c4b --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.*; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** Shared expression compiler for derived fields and metrics. Target emission follows semantic checks. */ +final class ExpressionCompiler { + private static final List DIALECTS = List.of("TABLEAU", "SNOWFLAKE", "ANSI_SQL"); + private ExpressionCompiler() {} + + enum Level { CONSTANT, ROW, AGGREGATE } + record Selected(String text, String dialect) {} + record Parsed(ExpressionAst.Node root, String dialect) {} + record Reference(List parts, boolean tableau) { + Reference { parts = List.copyOf(parts); } + } + record Binding(String expression, String datatype, Set datasets, Level level) { + Binding { datasets = Set.copyOf(datasets); } + Binding(String expression, String datatype, String dataset, Level level) { + this(expression, datatype, dataset == null ? Set.of() : Set.of(dataset), level); + } + Binding(String expression, String datatype, String dataset) { + this(expression, datatype, dataset, Level.ROW); + } + String dataset() { return datasets.size() == 1 ? datasets.iterator().next() : null; } + } + record Compiled(String expression, String datatype, Level level, Set datasets) { + Compiled { datasets = Set.copyOf(datasets); } + } + @FunctionalInterface interface ReferenceResolver { Binding resolve(Reference reference); } + + static Selected select(Map owner) { + Map expression = getMap(owner, "expression"); + List dialects = expression == null ? null : getList(expression, "dialects"); + if (dialects == null) { + throw new IllegalArgumentException("missing expression.dialects; provide TABLEAU, SNOWFLAKE or ANSI_SQL"); + } + Map candidates = new LinkedHashMap<>(); + for (Object entry : dialects) { + Map value = asMap(entry); + String dialect = getString(value, "dialect"); + if (DIALECTS.contains(dialect)) { + if (candidates.containsKey(dialect)) { + throw new IllegalArgumentException("ambiguous expression: multiple " + dialect + " entries"); + } + candidates.put(dialect, getString(value, "expression")); + } + } + String dialect = DIALECTS.stream().filter(candidates::containsKey).findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "no supported dialect; provide TABLEAU, SNOWFLAKE or ANSI_SQL")); + String text = candidates.get(dialect); + if (text == null || text.isBlank()) throw new IllegalArgumentException(dialect + " expression is empty"); + return new Selected(text, dialect); + } + + static Parsed parse(String text, String dialect) { + if (!DIALECTS.contains(dialect)) throw new IllegalArgumentException("unsupported expression dialect " + dialect); + if (text == null || text.isBlank()) throw new IllegalArgumentException(dialect + " expression is empty"); + List tokens = ExpressionTokens.tokenize(text, dialect); + ExpressionAst.Node root = dialect.equals("TABLEAU") + ? new TuaExpressionParser(tokens).parse() : SqlExpressionParser.parse(text, dialect); + return new Parsed(root, dialect); + } + + static Optional directReference(Parsed parsed) { + return parsed.root() instanceof ExpressionAst.Field field ? Optional.of(field.reference()) : Optional.empty(); + } + + static Compiled compile(Parsed parsed, ReferenceResolver resolver) { + ExpressionAst.Typed checked = new ExpressionAnalyzer(parsed.dialect(), resolver).analyze(parsed.root()); + String expression = new TuaExpressionEmitter().emit(checked); + return new Compiled(expression, checked.type().datatype, checked.level(), checked.datasets()); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java new file mode 100644 index 00000000..b5a585b2 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import java.util.Map; +import java.util.Set; + +/** Closed, version-controlled target capabilities; parser acceptance never implies function support. */ +final class ExpressionFunctionRegistry { + enum Rule { AGGREGATE, COALESCE, NULLIF, ISNULL, NUMERIC, YEAR, LENGTH, POSITION, SUBSTRING } + record Spec(String target, int min, int max, Rule rule, Set dialects) {} + private static final Set ALL = Set.of("SNOWFLAKE", "ANSI_SQL", "TABLEAU"); + private static final Set SQL = Set.of("SNOWFLAKE", "ANSI_SQL"); + private static final Set TUA = Set.of("TABLEAU"); + private static Spec spec(String target, int min, int max, Rule rule, Set dialects) { + return new Spec(target, min, max, rule, dialects); + } + private static final Map FUNCTIONS = Map.ofEntries( + Map.entry("SUM", spec("SUM", 1, 1, Rule.AGGREGATE, ALL)), + Map.entry("AVG", spec("AVG", 1, 1, Rule.AGGREGATE, ALL)), + Map.entry("MIN", spec("MIN", 1, 1, Rule.AGGREGATE, ALL)), + Map.entry("MAX", spec("MAX", 1, 1, Rule.AGGREGATE, ALL)), + Map.entry("COUNT", spec("COUNT", 1, 1, Rule.AGGREGATE, ALL)), + Map.entry("COUNTD", spec("COUNTD", 1, 1, Rule.AGGREGATE, TUA)), + Map.entry("COALESCE", spec("IFNULL", 2, Integer.MAX_VALUE, Rule.COALESCE, SQL)), + Map.entry("IFNULL", spec("IFNULL", 2, 2, Rule.COALESCE, TUA)), + Map.entry("NULLIF", spec("IF", 2, 2, Rule.NULLIF, SQL)), + Map.entry("ISNULL", spec("ISNULL", 1, 1, Rule.ISNULL, TUA)), + Map.entry("ABS", spec("ABS", 1, 1, Rule.NUMERIC, ALL)), + Map.entry("CEIL", spec("CEILING", 1, 1, Rule.NUMERIC, SQL)), + Map.entry("CEILING", spec("CEILING", 1, 1, Rule.NUMERIC, TUA)), + Map.entry("FLOOR", spec("FLOOR", 1, 1, Rule.NUMERIC, ALL)), + Map.entry("ROUND", spec("ROUND", 1, 2, Rule.NUMERIC, ALL)), + Map.entry("YEAR", spec("YEAR", 1, 1, Rule.YEAR, ALL)), + Map.entry("LENGTH", spec("LEN", 1, 1, Rule.LENGTH, SQL)), + Map.entry("LEN", spec("LEN", 1, 1, Rule.LENGTH, TUA)), + Map.entry("POSITION", spec("FIND", 2, 2, Rule.POSITION, SQL)), + Map.entry("FIND", spec("FIND", 2, 2, Rule.POSITION, TUA)), + Map.entry("SUBSTRING", spec("MID", 2, 3, Rule.SUBSTRING, SQL)), + Map.entry("MID", spec("MID", 2, 3, Rule.SUBSTRING, TUA))); + + private ExpressionFunctionRegistry() {} + static Spec require(String name, String dialect, int count, boolean distinct) { + Spec spec = FUNCTIONS.get(name); + if (spec == null) throw new IllegalArgumentException("unsupported function " + name); + if (!spec.dialects().contains(dialect)) { + throw new IllegalArgumentException(name + " is outside the supported " + dialect + " subset"); + } + if (distinct && (!name.equals("COUNT") || dialect.equals("TABLEAU"))) { + throw new IllegalArgumentException("DISTINCT is supported only by SQL COUNT(DISTINCT field)"); + } + if (count < spec.min() || count > spec.max()) { + throw new IllegalArgumentException(name + " expects " + + (spec.min() == spec.max() ? spec.min() : spec.min() + " to " + spec.max()) + " arguments"); + } + return spec; + } + static Spec get(String name) { return FUNCTIONS.get(name); } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java new file mode 100644 index 00000000..a39c1998 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** Bounded lexical preflight shared by the SQL and native Tua frontends. */ +final class ExpressionTokens { + enum Kind { WORD, IDENTIFIER, STRING, NUMBER, SYMBOL, END } + record Token(Kind kind, String text, int offset, boolean bracket) {} + private ExpressionTokens() {} + static List tokenize(String text, String dialect) { + if (text.length() > 32768) throw new IllegalArgumentException("expression exceeds 32768 characters"); + List tokens = new ArrayList<>(); + int nesting = 0; + for (int i = 0; i < text.length();) { + char c = text.charAt(i); + if (Character.isWhitespace(c)) { i++; continue; } + int start = i; + if (c == '\'' || c == '"' || c == '[') { + if (c == '[' && dialect.equals("SNOWFLAKE")) throw lexical(dialect, i, "use double-quoted SQL identifiers"); + boolean string = c == '\'' || (c == '"' && dialect.equals("TABLEAU")); + char end = c == '[' ? ']' : c; + StringBuilder value = new StringBuilder(); + boolean closed = false; + i++; + while (i < text.length()) { + char part = text.charAt(i++); + if (part == end) { + if (i < text.length() && text.charAt(i) == end) { value.append(end); i++; } + else { closed = true; break; } + } else { + if (Character.isISOControl(part) || (string && part == '\\')) { + throw lexical(dialect, i - 1, "control characters and backslash string escapes are unsupported"); + } + value.append(part); + } + } + if (!closed) throw lexical(dialect, start, "unterminated quoted value"); + tokens.add(new Token(string ? Kind.STRING : Kind.IDENTIFIER, value.toString(), start, c == '[')); + } else if (Character.isDigit(c) || (c == '.' && i + 1 < text.length() && Character.isDigit(text.charAt(i + 1)))) { + i++; + while (i < text.length() && (Character.isDigit(text.charAt(i)) || text.charAt(i) == '.')) i++; + if (i < text.length() && (text.charAt(i) == 'e' || text.charAt(i) == 'E')) { + i++; + if (i < text.length() && (text.charAt(i) == '+' || text.charAt(i) == '-')) i++; + while (i < text.length() && Character.isDigit(text.charAt(i))) i++; + } + tokens.add(new Token(Kind.NUMBER, text.substring(start, i), start, false)); + } else if (Character.isLetter(c) || c == '_') { + i++; + while (i < text.length() && (Character.isLetterOrDigit(text.charAt(i)) || text.charAt(i) == '_' || text.charAt(i) == '$')) i++; + tokens.add(new Token(Kind.WORD, text.substring(start, i), start, false)); + } else { + if (i + 1 < text.length() && (text.startsWith("--", i) || text.startsWith("/*", i))) { + throw lexical(dialect, i, "comments are unsupported in metric expressions"); + } + String symbol = String.valueOf(c); + if (i + 1 < text.length() && Set.of("<=", ">=", "<>", "!=").contains(text.substring(i, i + 2))) { + symbol = text.substring(i, i + 2); + i++; + } + if (!"()+-*/.,=<>!".contains(String.valueOf(c))) throw lexical(dialect, start, "unsupported character '" + c + "'"); + tokens.add(new Token(Kind.SYMBOL, symbol, start, false)); + i++; + } + Token added = tokens.get(tokens.size() - 1); + if (added.kind() == Kind.NUMBER) number(added.text()); + if (added.kind() == Kind.SYMBOL && added.text().equals("(") && ++nesting > 128) { + throw lexical(dialect, start, "expression nesting exceeds 128 levels"); + } + if (added.kind() == Kind.SYMBOL && added.text().equals(")")) nesting--; + if (tokens.size() > 8192) throw lexical(dialect, start, "too many expression tokens"); + } + if (nesting > 0) throw lexical(dialect, text.length(), "expected )"); + tokens.add(new Token(Kind.END, "end of expression", text.length(), false)); + return tokens; + } + + private static IllegalArgumentException lexical(String dialect, int offset, String message) { + return new IllegalArgumentException(dialect + " at character " + (offset + 1) + ": " + message); + } + static BigDecimal number(String text) { + BigDecimal value; + try { value = new BigDecimal(text); } + catch (NumberFormatException e) { throw new IllegalArgumentException("invalid numeric literal '" + text + "'"); } + if (Math.abs((long) value.scale()) > 1000 || value.precision() > 1000) { + throw new IllegalArgumentException("numeric literal is too large"); + } + return value; + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java new file mode 100644 index 00000000..071a94d7 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.getList; +import static org.apache.ossie.util.DataStructureUtils.getString; +import static org.apache.ossie.util.DataStructureUtils.streamMaps; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.ossie.converter.ExpressionCompiler.Binding; +import org.apache.ossie.converter.ExpressionCompiler.Compiled; +import org.apache.ossie.converter.ExpressionCompiler.Level; +import org.apache.ossie.converter.ExpressionCompiler.Parsed; +import org.apache.ossie.converter.ExpressionCompiler.Reference; +import org.apache.ossie.converter.ExpressionCompiler.Selected; +import org.apache.ossie.converter.MetricFieldResolver.Identifier; + +/** + * A per-conversion field dependency plan. SQL field expressions have dataset-local physical + * column scope plus explicitly declared semantic aliases; metric scope remains semantic only. + * Calculated aliases are expanded over direct semantic fields, never guessed global Tua names. + */ +final class FieldExpressionPlan { + record PlannedField(String dataset, String name, Map source, + Parsed parsed, Reference directReference, String physicalColumn, String calculatedApiName) { + boolean direct() { return physicalColumn != null; } + } + + private record Key(String dataset, String field) { + @Override public String toString() { return dataset + "." + field; } + } + + private final Map> datasets = new LinkedHashMap<>(); + private final Map fields = new LinkedHashMap<>(); + private final Map> byDataset = new LinkedHashMap<>(); + private final Map resolved = new HashMap<>(); + private final Map> lineage = new HashMap<>(); + private final Map>> sqlReferences = new HashMap<>(); + private final Map>> tableauReferences = new HashMap<>(); + private final Map>> qualifiers = new HashMap<>(); + private final Map> exported = new HashMap<>(); + private final Map target; + private final ArrayDeque pending = new ArrayDeque<>(); + private boolean indexed; + + FieldExpressionPlan(Map source, Map target) { + this.target = target; + Set reserved = new HashSet<>(); + for (String list : List.of("metrics", "semanticCalculatedDimensions", "semanticCalculatedMeasurements")) { + for (Map item : items(list.equals("metrics") ? source : target, list)) { + String name = getString(item, list.equals("metrics") ? "name" : "apiName"); + if (name != null) reserved.add(name.toUpperCase(Locale.ROOT)); + } + } + for (Map dataset : items(source, "datasets")) { + String datasetName = requiredName(dataset, "dataset"); + if (datasets.putIfAbsent(datasetName, dataset) != null) { + throw new IllegalArgumentException("Duplicate dataset declaration '" + datasetName + "'"); + } + byDataset.put(datasetName, new ArrayList<>()); + qualifiers.put(datasetName, sourceQualifiers(dataset)); + for (Map field : items(dataset, "fields")) { + String fieldName = requiredName(field, "field in dataset '" + datasetName + "'"); + Key key = new Key(datasetName, fieldName); + try { + Selected selected = ExpressionCompiler.select(field); + Parsed parsed = ExpressionCompiler.parse(selected.text(), selected.dialect()); + Reference reference = ExpressionCompiler.directReference(parsed).orElse(null); + // A Tua reference addresses semantic fields, not physical catalog columns. + String physical = reference != null && !reference.tableau() + ? physicalColumn(datasetName, reference) : null; + PlannedField planned = new PlannedField(datasetName, fieldName, field, parsed, + physical == null ? null : reference, physical, null); + if (fields.putIfAbsent(key, planned) != null) { + throw new IllegalArgumentException("Duplicate field declaration"); + } + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Field '" + key + "': " + e.getMessage(), e); + } + } + } + // Reserve every ordinary name before assigning suffixes, independent of declaration order. + Map candidateCounts = new HashMap<>(); + for (PlannedField field : fields.values()) { + if (!field.direct()) candidateCounts.merge(candidate(field).toUpperCase(Locale.ROOT), 1, Integer::sum); + } + for (Key key : fields.keySet().stream().sorted(Comparator.comparing(Key::dataset).thenComparing(Key::field)).toList()) { + PlannedField field = fields.get(key); + if (!field.direct()) { + String base = candidate(field); + String name = base; + if (candidateCounts.get(base.toUpperCase(Locale.ROOT)) > 1 + || reserved.contains(base.toUpperCase(Locale.ROOT))) { + String hash = digest(key.dataset() + "\u0000" + key.field()); + int length = 12; + do { + if (length > hash.length()) { + throw new IllegalArgumentException("Cannot allocate unique calculated field name for '" + key + "'"); + } + name = base + "__" + hash.substring(0, length); + length += 4; + } while (reserved.contains(name.toUpperCase(Locale.ROOT)) + || candidateCounts.containsKey(name.toUpperCase(Locale.ROOT))); + } + reserved.add(name.toUpperCase(Locale.ROOT)); + fields.put(key, new PlannedField(field.dataset(), field.name(), field.source(), field.parsed(), + null, null, name)); + } + } + for (PlannedField field : fields.values()) byDataset.get(field.dataset()).add(field); + byDataset.replaceAll((key, value) -> List.copyOf(value)); + for (PlannedField field : fields.values()) { + addIndex(tableauReferences, field.dataset(), field.name(), field); + addIndex(sqlReferences, field.dataset(), normalizeDeclaration(field.name()), field); + if (field.direct()) { + List parts = field.directReference().parts(); + addIndex(sqlReferences, field.dataset(), normalize(parts.get(parts.size() - 1)), field); + } + } + } + + List fields(String dataset) { return byDataset.getOrDefault(dataset, List.of()); } + + boolean isDirect(String dataset, String field) { return field(dataset, field).direct(); } + + String calculatedApiName(String dataset, String field) { return field(dataset, field).calculatedApiName(); } + + boolean hasPhysicalDependencies(String dataset, String field) { + resolve(dataset, field); + return !lineage.get(new Key(dataset, field)).isEmpty(); + } + + List> dependencies(String dataset, String field) { + resolve(dataset, field); + return lineage.get(new Key(dataset, field)).stream() + .sorted(Comparator.comparing(Key::dataset).thenComparing(Key::field)) + .map(key -> Map.of("dependentDefinitionApiName", key.dataset(), + "dependentFieldApiName", key.field())).toList(); + } + + /** Compile all derived fields, including unused ones, so unsupported fields cannot disappear. */ + void compileAll() { + indexExported(); + for (PlannedField field : fields.values()) { + if (!field.direct()) resolve(field.dataset(), field.name()); + } + } + + /** Exact declaration lookup for a metric resolver that has already resolved identifier spelling. */ + Binding resolve(String dataset, String field) { + indexExported(); + Key key = new Key(dataset, field); + Binding cached = resolved.get(key); + if (cached != null) return cached; + PlannedField planned = field(dataset, field); + if (planned.direct()) { + Map output = exported.get(key); + if (output == null) throw new IllegalArgumentException("Field '" + key + "' was not exported as a direct field"); + String datatype = getString(planned.source(), "datatype"); + String targetType = getString(output, "dataType"); + if (datatype == null || datatype.isBlank()) datatype = SalesforceDataTypeMapper.toOssie(targetType); + if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { + throw new IllegalArgumentException("Field '" + key + "' has no supported datatype; declare a portable field datatype"); + } + if (targetType == null || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { + throw new IllegalArgumentException("Field '" + key + "' datatype conflicts with exported Salesforce dataType '" + targetType + "'"); + } + Binding binding = new Binding(bracket(dataset) + "." + bracket(field), datatype, dataset, Level.ROW); + resolved.put(key, binding); + lineage.put(key, Set.of(key)); + return binding; + } + if (pending.contains(key)) { + throw new IllegalArgumentException("Calculated field dependency cycle: " + + String.join(" -> ", pending.stream().map(Key::toString).toList()) + " -> " + key); + } + if (pending.size() >= 128) { + throw new IllegalArgumentException("Calculated field dependency depth exceeds 128 at '" + key + "'"); + } + pending.addLast(key); + try { + Set dependencies = new LinkedHashSet<>(); + Compiled compiled = ExpressionCompiler.compile(planned.parsed(), reference -> bind(planned, reference, dependencies)); + if (compiled.level() == Level.AGGREGATE) { + throw new IllegalArgumentException("Dataset fields must be row expressions; declare aggregate expressions as metrics"); + } + String declared = getString(planned.source(), "datatype"); + String datatype = compiled.datatype(); + if (declared != null) { + if (!compatibleResult(declared, datatype)) { + throw new IllegalArgumentException("Declared datatype '" + declared + + "' conflicts with calculated result datatype '" + datatype + "'"); + } + datatype = declared; + } + if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { + throw new IllegalArgumentException("Calculated field needs a supported, unambiguous datatype"); + } + // Constant fields still belong to their declared dataset when consumed by a metric. + Binding binding = new Binding(compiled.expression(), datatype, dataset, Level.ROW); + resolved.put(key, binding); + lineage.put(key, Set.copyOf(dependencies)); + return binding; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Field '" + key + "': " + e.getMessage(), e); + } finally { + pending.removeLast(); + } + } + + private Binding bind(PlannedField owner, Reference reference, Set dependencies) { + List parts = reference.parts(); + String text = String.join(".", parts.stream().map(Identifier::text).toList()); + if (parts.isEmpty()) throw new IllegalArgumentException("Empty field reference"); + if (reference.tableau()) { + if (parts.size() != 2 || !owner.dataset().equals(parts.get(0).text())) { + throw new IllegalArgumentException("Row field reference '" + text + + "' must use [" + owner.dataset() + "].[declared field]; cross-dataset row calculations are unsupported"); + } + } else if (!validQualifier(owner.dataset(), parts.subList(0, parts.size() - 1))) { + throw new IllegalArgumentException("Unknown physical or semantic dataset qualifier in row field reference '" + text + "'"); + } + Identifier name = parts.get(parts.size() - 1); + Set matches = (reference.tableau() ? tableauReferences : sqlReferences) + .getOrDefault(owner.dataset(), Map.of()) + .getOrDefault(reference.tableau() ? name.text() : normalize(name), Set.of()); + if (matches.isEmpty()) { + throw new IllegalArgumentException("Unknown row field reference '" + text + + "'; declare its physical column or semantic field in dataset '" + owner.dataset() + "'"); + } + if (matches.size() > 1) { + throw new IllegalArgumentException("Ambiguous row field reference '" + text + + "' matches multiple declared physical columns or semantic fields"); + } + PlannedField match = matches.iterator().next(); + Binding binding = resolve(match.dataset(), match.name()); + dependencies.addAll(lineage.get(new Key(match.dataset(), match.name()))); + return binding; + } + + private void indexExported() { + if (indexed) return; + for (Map dataset : items(target, "semanticDataObjects")) { + String name = getString(dataset, "apiName"); + for (String kind : List.of("semanticDimensions", "semanticMeasurements")) { + for (Map item : items(dataset, kind)) { + Key key = new Key(name, getString(item, "apiName")); + if (exported.putIfAbsent(key, item) != null) { + throw new IllegalArgumentException("Duplicate exported field '" + key + "'"); + } + } + } + } + indexed = true; + } + + private PlannedField field(String dataset, String field) { + PlannedField planned = fields.get(new Key(dataset, field)); + if (planned == null) throw new IllegalArgumentException("Unknown declared field '" + dataset + "." + field + "'"); + return planned; + } + + private String physicalColumn(String dataset, Reference reference) { + List parts = reference.parts(); + if (parts.isEmpty() || !validQualifier(dataset, parts.subList(0, parts.size() - 1))) { + throw new IllegalArgumentException("Direct field qualifier does not match its declared dataset or physical source"); + } + return parts.get(parts.size() - 1).text(); + } + + private boolean validQualifier(String dataset, List qualifier) { + return qualifier.isEmpty() || qualifiers.get(dataset).contains(qualifier.stream() + .map(FieldExpressionPlan::normalize).toList()); + } + + private static Set> sourceQualifiers(Map dataset) { + Set> result = new HashSet<>(); + result.add(List.of(normalizeDeclaration(getString(dataset, "name")))); + String source = getString(dataset, "source"); + if (source != null) { + try { + Reference ref = ExpressionCompiler.directReference(ExpressionCompiler.parse(source, "ANSI_SQL")).orElse(null); + if (ref != null) { + List parts = ref.parts().stream().map(FieldExpressionPlan::normalize).toList(); + for (int i = 0; i < parts.size(); i++) result.add(List.copyOf(parts.subList(i, parts.size()))); + } + } catch (IllegalArgumentException ignored) { + // Opaque external source identifiers do not create implicit SQL aliases. + } + } + return Set.copyOf(result); + } + + private static void addIndex(Map>> index, + String dataset, String name, PlannedField field) { + index.computeIfAbsent(dataset, key -> new HashMap<>()) + .computeIfAbsent(name, key -> new LinkedHashSet<>()).add(field); + } + + private static boolean compatibleResult(String declared, String inferred) { + if (SalesforceDataTypeMapper.toSalesforce(declared) == null) return false; + if (inferred == null || declared.equals(inferred)) return true; + return Set.of("Decimal", "Float").contains(declared) + && Set.of("Integer", "Decimal", "Float").contains(inferred); + } + + private static String candidate(PlannedField field) { + String name = (field.dataset() + "__" + field.name()).replaceAll("[^A-Za-z0-9_]", "_"); + if (name.isEmpty() || Character.isDigit(name.charAt(0))) name = "field_" + name; + return name; + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is required by the Java runtime", e); + } + } + + private static String bracket(String name) { + if (name == null || name.isBlank() || name.contains("[") || name.contains("]") + || name.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException("Semantic name '" + name + "' cannot be represented safely in Tua"); + } + return "[" + name + "]"; + } + + private static String normalize(Identifier name) { + return name.quoted() ? name.text() : name.text().toUpperCase(Locale.ROOT); + } + + private static String normalizeDeclaration(String name) { + if (name.startsWith("\"") && name.endsWith("\"") && name.length() >= 2) { + String text = name.substring(1, name.length() - 1); + if (text.isEmpty() || text.replace("\"\"", "").contains("\"")) { + throw new IllegalArgumentException("Invalid quoted declaration name '" + name + "'"); + } + return text.replace("\"\"", "\""); + } + return name.toUpperCase(Locale.ROOT); + } + + private static String requiredName(Map item, String kind) { + String name = getString(item, "name"); + if (name == null || name.isBlank()) throw new IllegalArgumentException("Missing name for " + kind); + return name; + } + + private static List> items(Map map, String name) { + List list = getList(map, name); + return list == null ? List.of() : streamMaps(list).toList(); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java index 4147e0ef..f2d79763 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java @@ -24,7 +24,6 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.regex.Pattern; import static org.apache.ossie.converter.ConverterConstants.*; import static org.apache.ossie.util.DataStructureUtils.*; @@ -44,27 +43,9 @@ public class FieldMappingHandler implements PipelineStep { private static final Set SF_FIELD_HANDLED_PROPS = Set.of(API_NAME, LABEL, DESCRIPTION, DATA_OBJECT_FIELD_NAME); - // Compiled regex pattern for SQL keywords that indicate calculated expressions - private static final Pattern CALCULATED_KEYWORDS_PATTERN = Pattern.compile( - "\\b(CASE|WHEN|THEN|ELSE|END|CAST|CONVERT|EXTRACT|SUBSTRING|SUBSTR|" + - "COALESCE|NULLIF|IFNULL|CONCAT|UPPER|LOWER|TRIM|LENGTH|" + - "AND|OR|NOT|IN|BETWEEN|LIKE|IS\\s+NULL|IS\\s+NOT\\s+NULL|DISTINCT|" + - "COUNT|SUM|AVG|MIN|MAX|DATE|YEAR|MONTH|DAY)\\b" - ); - private final ConversionDirection direction; private final CustomExtensionHandler customExtensionHandler; - /** - * Enum representing the four possible field types in Salesforce Semantic Model. - */ - private enum FieldType { - DIMENSION, // Direct dimension: !isCalculated + hasDimension - MEASUREMENT, // Direct measurement: !isCalculated + !hasDimension - CALCULATED_DIMENSION, // Calculated dimension: isCalculated + hasDimension - CALCULATED_MEASUREMENT // Calculated measurement: isCalculated + !hasDimension - } - public FieldMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { this.direction = direction; this.customExtensionHandler = customExtensionHandler; @@ -75,38 +56,72 @@ public FieldMappingHandler(ConversionDirection direction, CustomExtensionHandler */ @Override public void execute(Map sourceData, Map outputData, Map mappings) { + execute(new ConversionContext(sourceData, outputData), mappings); + } + + @Override + public void execute(ConversionContext context, Map mappings) { logger.debug("Mapping fields in {} direction", direction); if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { - mapOssieToSalesforce(sourceData, outputData); + FieldExpressionPlan plan = new FieldExpressionPlan(context.sourceData(), context.outputData()); + mapOssieToSalesforce(context.sourceData(), context.outputData(), plan); + context.fieldPlan(plan); } else { - mapSalesforceToOssie(sourceData, outputData); + mapSalesforceToOssie(context.sourceData(), context.outputData()); } } - /** - * Maps Ossie dataset fields to Salesforce SemanticDimensions and SemanticMeasurements. - * - * @param outputData The output map containing semanticModel - * @param sourceData The source Ossie data - */ - private void mapOssieToSalesforce( - Map sourceData, Map outputData) { - - List ossieDatasets = getList(sourceData, DATASETS); - - List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); - - for (Object ossieDatasetObj : ossieDatasets) { - Map ossieDataset = asMap(ossieDatasetObj); - - String datasetName = getString(ossieDataset, NAME); - if (datasetName == null) continue; - - // Find matching SemanticDataObject - Map sfDataObject = findItemById(sfDataObjects, API_NAME, datasetName); - if (sfDataObject == null) continue; - - processFieldsForDataset(ossieDataset, sfDataObject, outputData); + /** Emit physical bindings first, then compile every derived field against those bindings. */ + private void mapOssieToSalesforce(Map sourceData, + Map outputData, FieldExpressionPlan plan) { + List targets = getList(outputData, SEMANTIC_DATA_OBJECTS); + Map> targetsByName = new LinkedHashMap<>(); + if (targets != null) { + for (Object item : targets) { + Map target = asMap(item); + String name = getString(target, API_NAME); + if (targetsByName.putIfAbsent(name, target) != null) { + throw new IllegalArgumentException("Duplicate exported dataset '" + name + "'"); + } + } + } + for (Object item : getList(sourceData, DATASETS)) { + Map dataset = asMap(item); + String datasetName = getString(dataset, NAME); + Map target = targetsByName.get(datasetName); + if (target == null) { + throw new IllegalArgumentException("Dataset '" + datasetName + "' was not exported before field mapping"); + } + for (FieldExpressionPlan.PlannedField field : plan.fields(datasetName)) { + if (!field.direct()) continue; + Map sfField = mapFieldProperties(field.source(), field.physicalColumn()); + customExtensionHandler.restoreSalesforceCustomExtension(sfField, field.source()); + applyOssieDatatype(sfField, field.source()); + applyFieldDefaults(sfField); + getOrCreateList(target, field.source().containsKey(DIMENSION) + ? SEMANTIC_DIMENSIONS : SEMANTIC_MEASUREMENTS).add(sfField); + } + } + plan.compileAll(); + for (Object item : getList(sourceData, DATASETS)) { + String datasetName = getString(asMap(item), NAME); + for (FieldExpressionPlan.PlannedField field : plan.fields(datasetName)) { + if (field.direct()) continue; + ExpressionCompiler.Binding binding = plan.resolve(datasetName, field.name()); + Map calc = createSemanticCalculatedDimension(field.source(), binding.expression()); + calc.put(API_NAME, field.calculatedApiName()); + calc.put(DEPENDENCIES, plan.dependencies(datasetName, field.name())); + customExtensionHandler.restoreSalesforceCustomExtension(calc, field.source()); + applyOssieDatatype(calc, field.source()); + String exactType = getString(calc, DATA_TYPE); + if (exactType != null && !SalesforceDataTypeMapper.areCompatible(binding.datatype(), exactType)) { + throw new IllegalArgumentException("Field '" + datasetName + "." + field.name() + + "' calculated datatype conflicts with Salesforce extension dataType '" + exactType + "'"); + } + calc.putIfAbsent(DATA_TYPE, SalesforceDataTypeMapper.toSalesforce(binding.datatype())); + applyFieldDefaults(calc); + getOrCreateList(outputData, SEMANTIC_CALCULATED_DIMENSIONS).add(calc); + } } } @@ -193,7 +208,7 @@ private Map convertDimensionToOssieField(Map sfD // Wrap dataObjectFieldName in expression structure String dataObjectFieldName = getString(sfDimension, DATA_OBJECT_FIELD_NAME); if (dataObjectFieldName != null) { - ossieField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); + ossieField.put(EXPRESSION, wrapPhysicalExpression(dataObjectFieldName)); } // Store unmapped properties in custom_extensions @@ -212,7 +227,7 @@ private Map convertMeasurementToOssieField(Map s String dataObjectFieldName = getString(sfMeasurement, DATA_OBJECT_FIELD_NAME); if (dataObjectFieldName != null) { - ossieField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); + ossieField.put(EXPRESSION, wrapPhysicalExpression(dataObjectFieldName)); } // Store unmapped properties in custom_extensions @@ -263,83 +278,12 @@ private Map wrapExpression(String expressionValue) { return expression; } - /** - * Processes all fields for a single dataset. - * - *

Routing Logic: - * - * - * - * - * - *
Expression TypeHas dimension?Routes To
DirectYesdataObject.semanticDimensions
DirectNodataObject.semanticMeasurements
CalculatedN/AMODEL.semanticCalculatedDimensions
- * - * @param ossieDataset The Ossie dataset - * @param sfDataObject The Salesforce data object to add direct fields to - * @param outputData The Salesforce model for adding calculated dimensions - */ - private void processFieldsForDataset( - Map ossieDataset, Map sfDataObject, Map outputData) { - List ossieFields = getList(ossieDataset, FIELDS); - if (ossieFields == null) { - return; - } - - List sfDimensions = getList(sfDataObject, SEMANTIC_DIMENSIONS); - List sfMeasurements = getList(sfDataObject, SEMANTIC_MEASUREMENTS); - - for (Object ossieFieldObj : ossieFields) { - Map ossieField = asMap(ossieFieldObj); - - // Determine field type based on Ossie structure - boolean hasDimension = ossieField.containsKey(DIMENSION); - ExpressionInfo expressionInfo = unwrapExpression(ossieField); - - String expression = expressionInfo.expression(); - String dialect = expressionInfo.dialect(); - - // Skip calculated fields for non-Tableau dialects till we agree on a common dialect. - if (!DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression)) { - continue; - } - - // Check if this is a calculated field (Tableau dialect with calculated expression) - boolean isCalculated = DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression); - - if (isCalculated) { - // Create a semantic calculated dimension - Map calcDim = createSemanticCalculatedDimension(ossieField, expression); - - customExtensionHandler.restoreSalesforceCustomExtension(calcDim, ossieField); - - applyOssieDatatype(calcDim, ossieField); - - applyFieldDefaults(calcDim); - - // Add to semantic calculated dimensions array - List calcDimensions = getOrCreateList(outputData, SEMANTIC_CALCULATED_DIMENSIONS); - calcDimensions.add(calcDim); - } else { - // Non calculated field - add to data object - FieldType fieldType = hasDimension? FieldType.DIMENSION : FieldType.MEASUREMENT; - - Map sfField = mapFieldProperties(ossieField, expression); - - customExtensionHandler.restoreSalesforceCustomExtension(sfField, ossieField); - - applyOssieDatatype(sfField, ossieField); - - applyFieldDefaults(sfField); - - RoutingResult result = - routeFieldToArray(sfField, fieldType, sfDataObject, sfDimensions, sfMeasurements); - sfDimensions = result.dataObjectDimensions(); - sfMeasurements = result.dataObjectMeasurements(); - } - } + /** Physical columns are SQL identifiers, even when they contain operators or spaces. */ + private Map wrapPhysicalExpression(String column) { + return Map.of(DIALECTS, List.of(Map.of(DIALECT, "ANSI_SQL", EXPRESSION, + "\"" + column.replace("\"", "\"\"") + "\""))); } - /** * Maps field properties based on whether the field is calculated. * Includes common properties plus type-specific properties. @@ -400,143 +344,12 @@ private Map createSemanticCalculatedDimension( } // Set syntax for Tableau expressions - calcDim.put("syntax", DIALECT_TABLEAU); + calcDim.put("syntax", "Tua"); + calcDim.put("level", "Row"); return calcDim; } - /** - * Routes a field to the appropriate array based on its type. - * Initializes arrays lazily using computeIfAbsent. - * - * @param sfField The Salesforce field to route - * @param fieldType The field type - * @param sfDataObject The data object (for data object-level arrays) - * @return Updated arrays for all levels - */ - private RoutingResult routeFieldToArray( - Map sfField, - FieldType fieldType, - Map sfDataObject, - List currentDataObjectDimensions, - List currentDataObjectMeasurements) { - - List dataObjectDimensions = currentDataObjectDimensions; - List dataObjectMeasurements = currentDataObjectMeasurements; - - switch (fieldType) { - case DIMENSION: - dataObjectDimensions = getOrCreateList(sfDataObject, SEMANTIC_DIMENSIONS); - dataObjectDimensions.add(sfField); - break; - - case MEASUREMENT: - dataObjectMeasurements = getOrCreateList(sfDataObject, SEMANTIC_MEASUREMENTS); - dataObjectMeasurements.add(sfField); - break; - } - - return new RoutingResult(dataObjectDimensions, dataObjectMeasurements); - } - - /** - * Helper record to return updated data object arrays. - */ - private record RoutingResult(List dataObjectDimensions, List dataObjectMeasurements) {} - - /** - * Helper record to return expression value along with its dialect type. - */ - private record ExpressionInfo(String expression, String dialect) {} - - /** - * Extracts the expression value and dialect from Ossie field's expression.dialects[0].expression. - * This unwraps the nested structure to get the simple column reference and its dialect. - * - * @param ossieField The Ossie field containing expression structure - * @return ExpressionInfo containing the expression string and dialect type, or null if not found - */ - private ExpressionInfo unwrapExpression(Map ossieField) { - Object expressionObj = ossieField.get(EXPRESSION); - - Map expression = asMap(expressionObj); - Object dialectsObj = expression.get(DIALECTS); - - List dialects = asList(dialectsObj); - - Object selectedDialectObj = null; - for (Object dialectObj : dialects) { - Map dialect = asMap(dialectObj); - String dialectType = getString(dialect, DIALECT); - if (DIALECT_TABLEAU.equals(dialectType)) { - selectedDialectObj = dialectObj; - break; - } - } - - if (selectedDialectObj == null) { - selectedDialectObj = dialects.get(0); - } - - Map selectedDialect = asMap(selectedDialectObj); - Object expressionValue = selectedDialect.get(EXPRESSION); - String dialectType = getString(selectedDialect, DIALECT); - - return new ExpressionInfo((String) expressionValue, dialectType); - } - - /** - * Determines if an expression is calculated or a direct column reference. - * - *

A calculated expression contains: - *

    - *
  • SQL functions: CONCAT(), SUM(), CAST(), etc.
    • - *
  • Operators: +, -, *, /, %, ||
    • - *
  • SQL keywords: CASE, WHEN, AND, OR, etc.
    • - *
  • Comparisons: {@literal >, <, =, !=, <>}
    • - *
- * - *

A direct reference is a simple column name (possibly table-qualified): - *

    - *
  • customer_name
    • - *
  • customers.customer_name
    • - *
  • schema.table.column
    • - *
- * - * @param expression The SQL expression to evaluate - * @return true if calculated, false if direct reference - */ - private boolean isCalculatedExpression(String expression) { - if (expression == null || expression.isEmpty()) { - return false; - } - - String normalized = expression.trim().toUpperCase(); - - // Check for function calls (presence of parentheses) - if (normalized.contains("(") || normalized.contains("[")) { - return true; - } - - // Check for operators (arithmetic, comparison, string concatenation) - if (normalized.contains("*") || normalized.contains("/") || normalized.contains("%") || - normalized.contains("||") || normalized.contains("::") || - normalized.contains(">") || normalized.contains("<") || - normalized.contains("!=") || normalized.contains("<>")) { - return true; - } - - // Check for arithmetic/comparison operators with spaces (avoid false positives like "customer-id") - if (normalized.contains(" + ") || normalized.contains(" - ") || - normalized.contains(" * ") || normalized.contains(" / ") || - normalized.contains(" = ")) { - return true; - } - - // Check for SQL keywords using compiled pattern - return CALCULATED_KEYWORDS_PATTERN.matcher(normalized).find(); - } - /** * Applies default values for required Salesforce field properties. * Only sets defaults if the property is not already present. @@ -572,12 +385,9 @@ private void applyOssieDatatype(Map sfField, Map if (exactSalesforceDataType != null) { if (ossieDatatype != null && !SalesforceDataTypeMapper.areCompatible(ossieDatatype, exactSalesforceDataType)) { - logger.warn( - "Field '{}' has Ossie datatype '{}' that conflicts with exact Salesforce dataType '{}'; " - + "preserving the Salesforce extension value", - getString(ossieField, NAME), - ossieDatatype, - exactSalesforceDataType); + throw new IllegalArgumentException("Field '" + getString(ossieField, NAME) + + "' has Ossie datatype '" + ossieDatatype + + "' that conflicts with Salesforce extension dataType '" + exactSalesforceDataType + "'"); } return; } @@ -587,11 +397,9 @@ private void applyOssieDatatype(Map sfField, Map } if (mappedSalesforceDataType == null) { - logger.warn( - "Field '{}' has Ossie datatype '{}' with no safe Salesforce mapping; omitting dataType", - getString(ossieField, NAME), - ossieDatatype); - return; + throw new IllegalArgumentException("Field '" + getString(ossieField, NAME) + + "' has Ossie datatype '" + ossieDatatype + + "' with no safe Salesforce mapping; provide a compatible native type"); } sfField.put(DATA_TYPE, mappedSalesforceDataType); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java new file mode 100644 index 00000000..70bc1fd1 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ossie.exception.ConversionException; + +/** Per-model metric dependency plan. References are checked before aggregate formulas are inlined. */ +final class MetricCompilationPlan { + private static final int MAX_DEPENDENCY_DEPTH = 128; + private final MetricFieldResolver fields; + private final Map> declarations = new LinkedHashMap<>(); + private final Map> sqlNames = new LinkedHashMap<>(); + private final Map compiled = new LinkedHashMap<>(); + private final Set visiting = new LinkedHashSet<>(); + + MetricCompilationPlan(Map source, MetricFieldResolver fields) { + this.fields = fields; + List metrics = getList(source, "metrics"); + if (metrics == null) return; + for (Object value : metrics) { + Map metric = asMap(value); + String name = getString(metric, "name"); + if (name == null || name.isBlank()) throw new ConversionException("Metric name must not be empty"); + if (declarations.putIfAbsent(name, metric) != null) { + throw new ConversionException("Metric '" + name + "': duplicate metric name"); + } + sqlNames.computeIfAbsent(MetricFieldResolver.normalizeDeclaration(name), key -> new ArrayList<>()).add(name); + } + } + + ExpressionCompiler.Compiled compile(String name) { + ExpressionCompiler.Compiled cached = compiled.get(name); + if (cached != null) return cached; + if (visiting.contains(name)) { + throw new ConversionException("Metric dependency cycle: " + String.join(" -> ", visiting) + " -> " + name); + } + if (visiting.size() >= MAX_DEPENDENCY_DEPTH) { + throw new ConversionException("Metric '" + name + "': dependency depth exceeds " + MAX_DEPENDENCY_DEPTH); + } + Map metric = declarations.get(name); + if (metric == null) throw new ConversionException("Unknown metric '" + name + "'"); + visiting.add(name); + try { + ExpressionCompiler.Compiled result = MetricExpressionTranslator.compile(metric, this::resolve); + try { fields.validateDatasets(result.datasets()); } + catch (IllegalArgumentException e) { throw new ConversionException("Metric '" + name + "': " + e.getMessage(), e); } + compiled.put(name, result); + return result; + } finally { + visiting.remove(name); + } + } + + private ExpressionCompiler.Binding resolve(ExpressionCompiler.Reference reference) { + if (reference.parts().size() == 1) { + MetricFieldResolver.Identifier identifier = reference.parts().get(0); + List matches = reference.tableau() + ? (declarations.containsKey(identifier.text()) ? List.of(identifier.text()) : List.of()) + : sqlNames.getOrDefault(MetricFieldResolver.normalize(identifier), List.of()); + if (!matches.isEmpty()) { + if (matches.size() != 1 || fields.hasUnqualifiedField(identifier, reference.tableau())) { + throw new IllegalArgumentException("ambiguous field or metric reference '" + identifier.text() + + "'; qualify the field or use a unique metric name"); + } + ExpressionCompiler.Compiled metric = compile(matches.get(0)); + return new ExpressionCompiler.Binding("(" + metric.expression() + ")", metric.datatype(), + metric.datasets(), metric.level()); + } + } + return fields.resolveBinding(reference); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java index 14caf724..25360f3f 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java @@ -19,545 +19,60 @@ package org.apache.ossie.converter; -import static org.apache.ossie.util.DataStructureUtils.*; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; +import static org.apache.ossie.util.DataStructureUtils.getString; import java.util.Map; -import java.util.Set; import org.apache.ossie.exception.ConversionException; -/** - * Compiles the supported metric-expression subset to Salesforce's Tua grammar. - * - *

This is deliberately an expression parser, not a SQL statement parser. Each - * production returns a typed formula with its aggregation level; unsupported - * syntax cannot fall through as untranslated text. See the README for the - * relationship to the proposed Python ossie_sql engine and extension points. - */ +/** Applies measurement-specific constraints around the shared expression compiler. */ final class MetricExpressionTranslator { - private static final List DIALECTS = List.of("TABLEAU", "SNOWFLAKE", "ANSI_SQL"); - private static final Set AGGREGATES = Set.of("SUM", "AVG", "MIN", "MAX", "COUNT", "COUNTD"); - record Result(String expression, String dataType) {} - private MetricExpressionTranslator() {} static Result translate(Map metric, Map sourceModel, Map targetModel) { + return translate(metric, new MetricFieldResolver(sourceModel, targetModel)); + } + + static Result translate(Map metric, MetricFieldResolver resolver) { + ExpressionCompiler.Compiled result = compile(metric, resolver::resolveBinding); + try { + resolver.validateDatasets(result.datasets()); + } catch (IllegalArgumentException e) { + throw new ConversionException("Metric '" + getString(metric, "name") + "': " + e.getMessage(), e); + } + return new Result(result.expression(), "Number"); + } + + static ExpressionCompiler.Compiled compile(Map metric, + ExpressionCompiler.ReferenceResolver references) { String name = getString(metric, "name"); try { - Map expression = getMap(metric, "expression"); - List dialects = expression == null ? null : getList(expression, "dialects"); - if (dialects == null) { - throw new IllegalArgumentException("missing expression.dialects; provide TABLEAU, SNOWFLAKE or ANSI_SQL"); - } - Map candidates = new java.util.LinkedHashMap<>(); - for (Object entry : dialects) { - Map value = asMap(entry); - String dialect = getString(value, "dialect"); - if (DIALECTS.contains(dialect)) { - if (candidates.containsKey(dialect)) { - throw new IllegalArgumentException("ambiguous expression: multiple " + dialect + " entries"); - } - candidates.put(dialect, getString(value, "expression")); - } - } - String dialect = DIALECTS.stream().filter(candidates::containsKey).findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "no supported dialect; provide TABLEAU, SNOWFLAKE or ANSI_SQL")); - String text = candidates.get(dialect); - if (text == null || text.isBlank()) { - throw new IllegalArgumentException(dialect + " expression is empty"); - } - MetricFieldResolver resolver = new MetricFieldResolver(sourceModel, targetModel); - Parser parser = new Parser(text, dialect, resolver); - Value result = parser.parse(); - resolver.validateDatasets(result.datasets); - Type declared = type(getString(metric, "datatype")); - if (metric.containsKey("datatype") && declared == Type.UNKNOWN) { + ExpressionCompiler.Selected selected = ExpressionCompiler.select(metric); + ExpressionCompiler.Compiled result = ExpressionCompiler.compile( + ExpressionCompiler.parse(selected.text(), selected.dialect()), references); + ExpressionAst.Type actual = result.datatype() == null ? ExpressionAst.Type.NULL : ExpressionAst.Type.of(result.datatype()); + ExpressionAst.Type declared = ExpressionAst.Type.of(getString(metric, "datatype")); + if (metric.containsKey("datatype") && declared == ExpressionAst.Type.UNKNOWN) { throw new IllegalArgumentException("unsupported metric datatype " + getString(metric, "datatype")); } - if (!result.type.numeric() && result.type != Type.NULL) { - throw new IllegalArgumentException("calculated measurements must be numeric, found " + result.type); + if (!actual.numeric() && actual != ExpressionAst.Type.NULL) { + throw new IllegalArgumentException("calculated measurements must be numeric, found " + actual); } - if (declared != Type.UNKNOWN && (!declared.numeric() - || (declared == Type.INTEGER && result.type != Type.INTEGER && result.type != Type.NULL))) { - throw new IllegalArgumentException("datatype " + getString(metric, "datatype") - + " is incompatible with expression result " + result.type); + if (declared != ExpressionAst.Type.UNKNOWN && (!declared.numeric() + || declared == ExpressionAst.Type.INTEGER && actual != ExpressionAst.Type.INTEGER && actual != ExpressionAst.Type.NULL)) { + throw new IllegalArgumentException("datatype " + getString(metric, "datatype") + " is incompatible with expression result " + actual); } - if (result.type == Type.NULL && !declared.numeric()) { + if (actual == ExpressionAst.Type.NULL && !declared.numeric()) { throw new IllegalArgumentException("all-null result needs an explicit numeric datatype"); } - if (result.level == Level.ROW) { + if (result.level() == ExpressionCompiler.Level.ROW) { throw new IllegalArgumentException("unaggregated field in metric; use an explicit aggregate"); } - return new Result(result.formula, "Number"); + return actual == ExpressionAst.Type.NULL + ? new ExpressionCompiler.Compiled(result.expression(), declared.datatype, result.level(), result.datasets()) + : result; } catch (IllegalArgumentException e) { throw new ConversionException("Metric '" + name + "': " + e.getMessage(), e); } } - - private enum Level { CONSTANT, ROW, AGGREGATE } - - private enum Type { - INTEGER, DECIMAL, FLOAT, STRING, BOOLEAN, DATE, DATETIME, DATETIME_TZ, NULL, UNKNOWN; - boolean numeric() { return this == INTEGER || this == DECIMAL || this == FLOAT; } - } - - private static Type type(String datatype) { - if (datatype == null) return Type.UNKNOWN; - return switch (datatype) { - case "Integer" -> Type.INTEGER; - case "Decimal" -> Type.DECIMAL; - case "Float" -> Type.FLOAT; - case "String" -> Type.STRING; - case "Boolean" -> Type.BOOLEAN; - case "Date" -> Type.DATE; - case "DateTime" -> Type.DATETIME; - case "DateTimeTz" -> Type.DATETIME_TZ; - default -> Type.UNKNOWN; - }; - } - - /** A synthesized parser attribute, not a second general-purpose expression model. */ - private record Value(String formula, Type type, Level level, Set datasets, - boolean field, BigDecimal number) { - Value(String formula, Type type, Level level, Set datasets) { - this(formula, type, level, datasets, false, null); - } - } - - private enum Kind { WORD, IDENTIFIER, STRING, NUMBER, SYMBOL, END } - private record Token(Kind kind, String text, int offset, boolean bracket) {} - - private static final class Parser { - private final List tokens; - private final String dialect; - private final MetricFieldResolver resolver; - private int position; - private int depth; - - Parser(String expression, String dialect, MetricFieldResolver resolver) { - this.dialect = dialect; - this.resolver = resolver; - this.tokens = tokenize(expression, dialect); - } - - Value parse() { - Value value = expression(); - if (peek().kind != Kind.END) throw error("unsupported or unexpected token '" + peek().text + "'"); - return value; - } - - private Value expression() { - if (++depth > 128) throw error("expression nesting exceeds 128 levels"); - Value value = or(); - depth--; - return value; - } - - private Value or() { - Value value = and(); - while (take("OR")) value = binary("OR", value, and(), Type.BOOLEAN); - return value; - } - - private Value and() { - Value value = not(); - while (take("AND")) value = binary("AND", value, not(), Type.BOOLEAN); - return value; - } - - // SQL NOT binds below comparisons, unlike the old draft's precedence table. - private Value not() { - int count = 0; - while (take("NOT")) { - if (++count > 128) throw error("too many unary operators"); - } - Value value = comparison(); - for (int i = 0; i < count; i++) { - require(value, Type.BOOLEAN, "NOT"); - value = new Value("(NOT " + value.formula + ")", Type.BOOLEAN, value.level, value.datasets); - } - return value; - } - - private Value comparison() { - Value value = additive(); - if (take("IS")) { - if (dialect.equals("TABLEAU")) throw error("use ISNULL in TABLEAU expressions"); - boolean negated = take("NOT"); - expect("NULL"); - return new Value((negated ? "(NOT ISNULL(" : "ISNULL(") + value.formula - + (negated ? "))" : ")"), Type.BOOLEAN, value.level, value.datasets); - } - if (Set.of("=", "!=", "<>", "<", "<=", ">", ">=").contains(peek().text)) { - String operator = next().text; - Value right = additive(); - compatible(value.type, right.type, "comparison"); - if (!Set.of("=", "!=", "<>").contains(operator) - && (value.type == Type.BOOLEAN || right.type == Type.BOOLEAN)) { - throw error("ordered comparison requires numeric, text or temporal operands"); - } - return compose("(" + value.formula + " " + (operator.equals("<>") ? "!=" : operator) - + " " + right.formula + ")", Type.BOOLEAN, List.of(value, right)); - } - return value; - } - - private Value additive() { - Value value = multiplicative(); - while (at("+") || at("-")) { - String operator = next().text; - Value right = multiplicative(); - value = binary(operator, value, right, numericType(value, right, operator)); - } - return value; - } - - private Value multiplicative() { - Value value = unary(); - while (at("*") || at("/")) { - String operator = next().text; - Value right = unary(); - if (operator.equals("/") && right.number != null && right.number.signum() == 0) { - throw error("division by literal zero; use NULLIF(denominator, 0) for a nullable denominator"); - } - Type resultType = numericType(value, right, operator); - value = binary(operator, value, right, operator.equals("/") ? Type.DECIMAL : resultType); - } - return value; - } - - private Value unary() { - List signs = new ArrayList<>(); - while (at("+") || at("-")) { - if (signs.size() >= 128) throw error("too many unary operators"); - signs.add(next().text); - } - Value value = primary(); - for (int i = signs.size() - 1; i >= 0; i--) { - numeric(value, "unary " + signs.get(i)); - boolean negative = signs.get(i).equals("-"); - value = new Value(negative ? "(-" + value.formula + ")" : value.formula, - value.type, value.level, value.datasets, false, - value.number == null ? null : negative ? value.number.negate() : value.number); - } - return value; - } - - private Value primary() { - if (take("(")) { - Value value = expression(); - expect(")"); - return value; - } - if (at("CASE")) { - if (dialect.equals("TABLEAU")) throw error("searched CASE is SQL; use IF in TABLEAU"); - next(); - return conditional(false); - } - if (at("IF") && dialect.equals("TABLEAU")) { - next(); - return conditional(true); - } - if (take("NULL")) return new Value("NULL", Type.NULL, Level.CONSTANT, Set.of()); - if (at("TRUE") || at("FALSE")) { - return new Value(next().text.toUpperCase(Locale.ROOT), Type.BOOLEAN, Level.CONSTANT, Set.of()); - } - Token token = next(); - if (token.kind == Kind.NUMBER) { - BigDecimal number; - try { number = new BigDecimal(token.text); } - catch (NumberFormatException e) { throw error("invalid numeric literal '" + token.text + "'"); } - if (Math.abs((long) number.scale()) > 1000 || number.precision() > 1000) { - throw error("numeric literal is too large"); - } - Type datatype = number.stripTrailingZeros().scale() <= 0 ? Type.INTEGER : Type.DECIMAL; - return new Value(number.toPlainString(), datatype, Level.CONSTANT, Set.of(), false, number); - } - if (token.kind == Kind.STRING) { - return new Value("'" + token.text.replace("'", "''") + "'", Type.STRING, Level.CONSTANT, Set.of()); - } - if (token.kind != Kind.WORD && token.kind != Kind.IDENTIFIER) { - throw error("expected a value, found '" + token.text + "'"); - } - if (take("(")) { - if (token.kind != Kind.WORD) throw error("quoted function names are unsupported"); - return function(token.text.toUpperCase(Locale.ROOT)); - } - List parts = new ArrayList<>(); - addIdentifier(parts, token); - boolean bracketed = token.bracket; - while (take(".")) { - Token part = next(); - if (bracketed != part.bracket) throw error("do not mix bracketed and SQL field identifiers"); - addIdentifier(parts, part); - } - // Existing ANSI_SQL fixtures use complete Tableau field notation. Keep - // that narrow compatibility spelling, with the same exact-name checks. - MetricFieldResolver.ResolvedField field = resolver.resolve(parts, bracketed); - return new Value(field.expression(), type(field.datatype()), Level.ROW, - Set.of(field.dataset()), true, null); - } - - private void addIdentifier(List parts, Token token) { - if (token.kind != Kind.WORD && token.kind != Kind.IDENTIFIER) throw error("expected field identifier"); - if (dialect.equals("TABLEAU") && !token.bracket) { - throw error("TABLEAU fields must use [dataset].[field] notation"); - } - parts.add(new MetricFieldResolver.Identifier(token.text, token.kind == Kind.IDENTIFIER)); - } - - private Value conditional(boolean tableau) { - List values = new ArrayList<>(); - StringBuilder formula = new StringBuilder("(IF "); - Type resultType = Type.NULL; - boolean first = true; - do { - if (!first) formula.append(" ELSEIF "); - if (!tableau) expect("WHEN"); - Value condition = expression(); - require(condition, Type.BOOLEAN, "conditional predicate"); - expect("THEN"); - Value branch = expression(); - resultType = compatible(resultType, branch.type, "conditional branches"); - values.add(condition); - values.add(branch); - formula.append(condition.formula).append(" THEN ").append(branch.formula); - first = false; - } while (tableau ? take("ELSEIF") : at("WHEN")); - Value otherwise = take("ELSE") ? expression() : new Value("NULL", Type.NULL, Level.CONSTANT, Set.of()); - expect("END"); - resultType = compatible(resultType, otherwise.type, "conditional branches"); - values.add(otherwise); - formula.append(" ELSE ").append(otherwise.formula).append(" END)"); - return compose(formula.toString(), resultType, values); - } - - private Value function(String name) { - if (++depth > 128) throw error("expression nesting exceeds 128 levels"); - boolean distinct = take("DISTINCT"); - if (at("*")) throw error("COUNT(*) is unsupported; name a declared field to count"); - List arguments = new ArrayList<>(); - if (!at(")")) { - do { arguments.add(expression()); } while (take(",")); - } - expect(")"); - depth--; - if (distinct && (!name.equals("COUNT") || dialect.equals("TABLEAU"))) { - throw error("DISTINCT is supported only by SQL COUNT(DISTINCT field)"); - } - if (AGGREGATES.contains(name)) { - if (name.equals("COUNTD") && !dialect.equals("TABLEAU")) throw error("use COUNT(DISTINCT field) in SQL"); - arity(name, arguments, 1, 1); - Value argument = arguments.get(0); - if (argument.level == Level.AGGREGATE) throw error("nested aggregate " + name + " is unsupported"); - if (argument.datasets.isEmpty()) throw error(name + " needs a declared field to establish its dataset"); - if (argument.datasets.size() > 1) throw error("one aggregate cannot combine fields from multiple datasets"); - boolean count = name.equals("COUNT") || name.equals("COUNTD"); - if (count && !argument.field) throw error(name + " requires a declared field; counting expressions is unsupported"); - if (name.equals("MIN") || name.equals("MAX")) { - if (argument.type == Type.BOOLEAN || argument.type == Type.UNKNOWN) { - throw error(name + " requires numeric, text or temporal operands"); - } - } else if (!count) numeric(argument, name); - return new Value((distinct ? "COUNTD" : name) + "(" + argument.formula + ")", - count ? Type.INTEGER : name.equals("AVG") ? Type.DECIMAL : argument.type, - Level.AGGREGATE, argument.datasets); - } - if (Set.of("COALESCE", "NULLIF").contains(name) && dialect.equals("TABLEAU")) { - throw error(name + " is SQL syntax; use IFNULL or IF in TABLEAU"); - } - if (Set.of("IFNULL", "ISNULL", "CEILING").contains(name) && !dialect.equals("TABLEAU")) { - throw error(name + " is outside the supported SQL subset"); - } - return switch (name) { - case "COALESCE", "IFNULL" -> coalesce(name, arguments); - case "NULLIF" -> nullif(arguments); - case "ISNULL" -> { - arity(name, arguments, 1, 1); - yield compose(call(name, arguments), Type.BOOLEAN, arguments); - } - case "ABS", "CEIL", "CEILING", "FLOOR", "ROUND" -> numericFunction(name, arguments); - default -> throw error("unsupported function " + name); - }; - } - - private Value coalesce(String name, List arguments) { - arity(name, arguments, 2, name.equals("IFNULL") ? 2 : Integer.MAX_VALUE); - Type resultType = Type.NULL; - for (Value argument : arguments) resultType = compatible(resultType, argument.type, name + " arguments"); - String formula = arguments.get(arguments.size() - 1).formula; - for (int i = arguments.size() - 2; i >= 0; i--) formula = "IFNULL(" + arguments.get(i).formula + ", " + formula + ")"; - return compose(formula, resultType, arguments); - } - - private Value nullif(List arguments) { - arity("NULLIF", arguments, 2, 2); - Value left = arguments.get(0); - Value right = arguments.get(1); - compatible(left.type, right.type, "NULLIF arguments"); - return compose("(IF (" + left.formula + " = " + right.formula + ") THEN NULL ELSE " - + left.formula + " END)", left.type, arguments); - } - - private Value numericFunction(String name, List arguments) { - if (name.equals("CEIL") && dialect.equals("TABLEAU")) throw error("use CEILING in TABLEAU"); - arity(name, arguments, 1, name.equals("ROUND") ? 2 : 1); - Value value = arguments.get(0); - numeric(value, name); - if (arguments.size() == 2) { - BigDecimal places = arguments.get(1).number; - if (places == null || places.stripTrailingZeros().scale() > 0 - || places.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0 - || places.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { - throw error("ROUND precision must be a 32-bit integer literal"); - } - } - String target = name.equals("CEIL") ? "CEILING" : name; - return compose(call(target, arguments), value.type, arguments); - } - - private String call(String name, List arguments) { - return name + "(" + String.join(", ", arguments.stream().map(Value::formula).toList()) + ")"; - } - - private void arity(String name, List arguments, int min, int max) { - if (arguments.size() < min || arguments.size() > max) { - throw error(name + " expects " + (min == max ? min : min + " to " + max) + " arguments"); - } - } - - private Value binary(String operator, Value left, Value right, Type resultType) { - if (operator.equals("AND") || operator.equals("OR")) { - require(left, Type.BOOLEAN, operator); - require(right, Type.BOOLEAN, operator); - } - return compose("(" + left.formula + " " + operator + " " + right.formula + ")", - resultType, List.of(left, right)); - } - - private Type numericType(Value left, Value right, String context) { - numeric(left, context); - numeric(right, context); - return compatible(left.type, right.type, context); - } - - private void numeric(Value value, String context) { - if (!value.type.numeric() && value.type != Type.NULL) { - throw error(context + " requires numeric operands, found " + value.type - + "; declare a compatible field datatype"); - } - } - - private void require(Value value, Type expected, String context) { - if (value.type != expected && value.type != Type.NULL) throw error(context + " requires " + expected + ", found " + value.type); - } - - private Type compatible(Type left, Type right, String context) { - if (left == Type.UNKNOWN || right == Type.UNKNOWN) throw error(context + " needs known field datatypes"); - if (left == Type.NULL) return right; - if (right == Type.NULL || left == right) return left; - if (left.numeric() && right.numeric()) { - if (left == Type.FLOAT || right == Type.FLOAT) return Type.FLOAT; - return Type.DECIMAL; - } - throw error(context + " has incompatible types " + left + " and " + right); - } - - private Value compose(String formula, Type type, List arguments) { - Level level = Level.CONSTANT; - Set datasets = new HashSet<>(); - for (Value argument : arguments) { - if (level != Level.CONSTANT && argument.level != Level.CONSTANT && level != argument.level) { - throw error("cannot mix aggregate and unaggregated field expressions"); - } - if (argument.level != Level.CONSTANT) level = argument.level; - datasets.addAll(argument.datasets); - } - // NULLIF duplicates its first operand. Bound expansion as well as input size. - if (formula.length() > 131072) throw error("translated expression exceeds 131072 characters"); - return new Value(formula, type, level, Set.copyOf(datasets)); - } - - private Token peek() { return tokens.get(position); } - private Token next() { Token token = peek(); if (token.kind != Kind.END) position++; return token; } - private boolean at(String text) { - return (peek().kind == Kind.WORD || peek().kind == Kind.SYMBOL) && peek().text.equalsIgnoreCase(text); - } - private boolean take(String text) { if (!at(text)) return false; next(); return true; } - private void expect(String text) { if (!take(text)) throw error("expected " + text + ", found '" + peek().text + "'"); } - private IllegalArgumentException error(String message) { - return new IllegalArgumentException(dialect + " at character " + (peek().offset + 1) + ": " + message); - } - } - - private static List tokenize(String text, String dialect) { - if (text.length() > 32768) throw new IllegalArgumentException("expression exceeds 32768 characters"); - List tokens = new ArrayList<>(); - for (int i = 0; i < text.length();) { - char c = text.charAt(i); - if (Character.isWhitespace(c)) { i++; continue; } - int start = i; - if (c == '\'' || c == '"' || c == '[') { - if (c == '[' && dialect.equals("SNOWFLAKE")) throw lexical(dialect, i, "use double-quoted SQL identifiers"); - boolean string = c == '\'' || (c == '"' && dialect.equals("TABLEAU")); - char end = c == '[' ? ']' : c; - StringBuilder value = new StringBuilder(); - boolean closed = false; - i++; - while (i < text.length()) { - char part = text.charAt(i++); - if (part == end) { - if (i < text.length() && text.charAt(i) == end) { value.append(end); i++; } - else { closed = true; break; } - } else { - if (Character.isISOControl(part) || (string && part == '\\')) { - throw lexical(dialect, i - 1, "control characters and backslash string escapes are unsupported"); - } - value.append(part); - } - } - if (!closed) throw lexical(dialect, start, "unterminated quoted value"); - tokens.add(new Token(string ? Kind.STRING : Kind.IDENTIFIER, value.toString(), start, c == '[')); - } else if (Character.isDigit(c) || (c == '.' && i + 1 < text.length() && Character.isDigit(text.charAt(i + 1)))) { - i++; - while (i < text.length() && (Character.isDigit(text.charAt(i)) || text.charAt(i) == '.')) i++; - if (i < text.length() && (text.charAt(i) == 'e' || text.charAt(i) == 'E')) { - i++; - if (i < text.length() && (text.charAt(i) == '+' || text.charAt(i) == '-')) i++; - while (i < text.length() && Character.isDigit(text.charAt(i))) i++; - } - tokens.add(new Token(Kind.NUMBER, text.substring(start, i), start, false)); - } else if (Character.isLetter(c) || c == '_') { - i++; - while (i < text.length() && (Character.isLetterOrDigit(text.charAt(i)) || text.charAt(i) == '_' || text.charAt(i) == '$')) i++; - tokens.add(new Token(Kind.WORD, text.substring(start, i), start, false)); - } else { - if (i + 1 < text.length() && (text.startsWith("--", i) || text.startsWith("/*", i))) { - throw lexical(dialect, i, "comments are unsupported in metric expressions"); - } - String symbol = String.valueOf(c); - if (i + 1 < text.length() && Set.of("<=", ">=", "<>", "!=").contains(text.substring(i, i + 2))) { - symbol = text.substring(i, i + 2); - i++; - } - if (!"()+-*/.,=<>!".contains(String.valueOf(c))) throw lexical(dialect, start, "unsupported character '" + c + "'"); - tokens.add(new Token(Kind.SYMBOL, symbol, start, false)); - i++; - } - if (tokens.size() > 8192) throw lexical(dialect, start, "too many expression tokens"); - } - tokens.add(new Token(Kind.END, "end of expression", text.length(), false)); - return tokens; - } - - private static IllegalArgumentException lexical(String dialect, int offset, String message) { - return new IllegalArgumentException(dialect + " at character " + (offset + 1) + ": " + message); - } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java index edda8a6e..2cff9226 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java @@ -19,197 +19,195 @@ package org.apache.ossie.converter; -import static org.apache.ossie.util.DataStructureUtils.getList; -import static org.apache.ossie.util.DataStructureUtils.getString; -import static org.apache.ossie.util.DataStructureUtils.streamMaps; +import static org.apache.ossie.util.DataStructureUtils.*; -import java.util.ArrayList; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; -/** Binds metric references to declared fields that the Salesforce converter actually exported. */ +/** Model-scoped indexes bind metric references to validated exported semantic identities. */ final class MetricFieldResolver { - record Identifier(String text, boolean quoted) {} - record ResolvedField(String expression, String datatype, String dataset) {} + private record Field(String dataset, String name, Map properties) {} - private record Field(Map dataset, Map field) {} - - private final List> datasets; - private final List> targetDatasets; - private final List> relationships; + private final NameIndex> datasets = new NameIndex<>(); + private final NameIndex unqualifiedFields = new NameIndex<>(); + private final Map> fieldsByDataset = new HashMap<>(); + private final Map>> targetDatasets; + private final Map>>> targetFields = new HashMap<>(); + private final Map>> targetCalculatedFields; + private final Map components; + private final FieldExpressionPlan fieldPlan; MetricFieldResolver(Map sourceModel, Map targetModel) { - datasets = items(sourceModel, "datasets"); - targetDatasets = items(targetModel, "semanticDataObjects"); - relationships = items(targetModel, "semanticRelationships"); + this(sourceModel, targetModel, null); } - /** Checks exported connectivity only; join grain and cardinality still require native validation. */ - void validateDatasets(Set referenced) { - if (referenced.size() < 2) { - return; - } - Map> graph = new HashMap<>(); - for (Map dataset : targetDatasets) { - String name = getString(dataset, "apiName"); - if (name != null) { - graph.put(name, new HashSet<>()); + MetricFieldResolver(Map sourceModel, Map targetModel, + FieldExpressionPlan fieldPlan) { + this.fieldPlan = fieldPlan; + for (Map dataset : items(sourceModel, "datasets")) { + String datasetName = getString(dataset, "name"); + datasets.add(datasetName, dataset); + NameIndex fields = fieldsByDataset.computeIfAbsent(datasetName, ignored -> new NameIndex<>()); + for (Map properties : items(dataset, "fields")) { + String name = getString(properties, "name"); + Field field = new Field(datasetName, name, properties); + fields.add(name, field); + unqualifiedFields.add(name, field); } } - for (Map relationship : relationships) { - if (Boolean.FALSE.equals(relationship.get("isEnabled"))) { - continue; - } - String left = getString(relationship, "leftSemanticDefinitionApiName"); - String right = getString(relationship, "rightSemanticDefinitionApiName"); - if (graph.containsKey(left) && graph.containsKey(right)) { - graph.get(left).add(right); - graph.get(right).add(left); - } + List> targets = items(targetModel, "semanticDataObjects"); + targetDatasets = index(targets, item -> getString(item, "apiName")); + for (Map dataset : targets) { + List> fields = new ArrayList<>(items(dataset, "semanticDimensions")); + fields.addAll(items(dataset, "semanticMeasurements")); + targetFields.put(getString(dataset, "apiName"), index(fields, item -> getString(item, "apiName"))); } - Set visited = new HashSet<>(); - ArrayDeque pending = new ArrayDeque<>(); - pending.add(referenced.iterator().next()); - while (!pending.isEmpty()) { - String dataset = pending.removeFirst(); - if (visited.add(dataset)) { - pending.addAll(graph.getOrDefault(dataset, Set.of())); - } - } - if (!visited.containsAll(referenced)) { + targetCalculatedFields = index(items(targetModel, "semanticCalculatedDimensions"), + item -> getString(item, "apiName")); + components = connectedComponents(targetDatasets.keySet(), items(targetModel, "semanticRelationships")); + } + + /** Connectivity is a prerequisite; it is not a proof of native join-grain equivalence. */ + void validateDatasets(Set referenced) { + if (referenced.size() < 2) return; + Integer component = components.get(referenced.iterator().next()); + if (component == null || referenced.stream().anyMatch(name -> !component.equals(components.get(name)))) { throw new IllegalArgumentException("Metric references disconnected datasets " + referenced.stream().sorted().collect(Collectors.joining(", ")) + "; declare supported relationships connecting them before exporting the metric"); } } + boolean hasUnqualifiedField(Identifier identifier, boolean tableau) { + return !unqualifiedFields.find(identifier, tableau).isEmpty(); + } + + ExpressionCompiler.Binding resolveBinding(ExpressionCompiler.Reference reference) { + return binding(reference.parts(), reference.tableau()); + } + ResolvedField resolve(List parts, boolean tableau) { + ExpressionCompiler.Binding value = binding(parts, tableau); + return new ResolvedField(value.expression(), value.datatype(), value.dataset()); + } + + private ExpressionCompiler.Binding binding(List parts, boolean tableau) { String reference = parts.stream().map(Identifier::text).collect(Collectors.joining(".")); if (parts.isEmpty() || parts.size() > 2) { throw new IllegalArgumentException("Reference '" + reference + "' must name a declared field or dataset.field; physical source paths are unsupported"); } if (tableau && parts.size() != 2) { - throw new IllegalArgumentException("TABLEAU field reference '" + reference - + "' must use [dataset].[field]"); + throw new IllegalArgumentException("TABLEAU field reference '" + reference + "' must use [dataset].[field]"); } - - List> candidates = datasets; + List candidates; if (parts.size() == 2) { - candidates = datasets.stream() - .filter(dataset -> matches(parts.get(0), getString(dataset, "name"), tableau)) - .toList(); - if (candidates.isEmpty()) { - throw new IllegalArgumentException("Unknown dataset in reference '" + reference - + "'; use a declared dataset name, not its physical source"); - } - if (candidates.size() > 1) { - throw new IllegalArgumentException("Ambiguous dataset in reference '" + reference - + "'; dataset declarations must have distinct names"); - } - } - - Identifier fieldName = parts.get(parts.size() - 1); - List fields = new ArrayList<>(); - for (Map dataset : candidates) { - for (Map field : items(dataset, "fields")) { - if (matches(fieldName, getString(field, "name"), tableau)) { - fields.add(new Field(dataset, field)); - } - } - } - if (fields.isEmpty()) { - throw new IllegalArgumentException("Unknown field reference '" + reference - + "'; declare the field under datasets[].fields before exporting the metric"); - } - if (fields.size() > 1) { - throw new IllegalArgumentException("Ambiguous field reference '" + reference - + "'; qualify the dataset and remove duplicate field declarations"); - } - - Field match = fields.get(0); - String datasetName = getString(match.dataset(), "name"); - // An unqualified field must not accidentally select one of two equivalent datasets. - if (datasets.stream().filter(dataset -> equivalentDeclaration( - datasetName, getString(dataset, "name"), tableau)).count() > 1) { + List> matches = datasets.find(parts.get(0), tableau); + if (matches.isEmpty()) throw new IllegalArgumentException("Unknown dataset in reference '" + reference + + "'; use a declared dataset name, not its physical source"); + if (matches.size() > 1) throw new IllegalArgumentException("Ambiguous dataset in reference '" + reference + + "'; dataset declarations must have distinct names"); + candidates = fieldsByDataset.get(getString(matches.get(0), "name")).find(parts.get(1), tableau); + } else { + candidates = unqualifiedFields.find(parts.get(0), tableau); + } + if (candidates.isEmpty()) throw new IllegalArgumentException("Unknown field reference '" + reference + + "'; declare the field under datasets[].fields before exporting the metric"); + if (candidates.size() > 1) throw new IllegalArgumentException("Ambiguous field reference '" + reference + + "'; qualify the dataset and remove duplicate field declarations"); + Field match = candidates.get(0); + if (datasets.declarations(match.dataset, tableau).size() > 1) { throw new IllegalArgumentException("Ambiguous dataset for reference '" + reference + "'; dataset declarations must have distinct names"); } - String sourceFieldName = getString(match.field(), "name"); - Map targetDataset = exportedItem(targetDatasets, datasetName, - "dataset", reference); - List> targetFields = new ArrayList<>(items(targetDataset, "semanticDimensions")); - targetFields.addAll(items(targetDataset, "semanticMeasurements")); - Map targetField = exportedItem(targetFields, sourceFieldName, "field", reference); - - String datatype = getString(match.field(), "datatype"); - String targetType = getString(targetField, "dataType"); - if (datatype == null || datatype.isBlank()) { - datatype = SalesforceDataTypeMapper.toOssie(targetType); + Map targetDataset = exported(targetDatasets, match.dataset, "dataset", reference); + if (fieldPlan != null && !fieldPlan.isDirect(match.dataset, match.name)) { + Map targetField = exported(targetCalculatedFields, + fieldPlan.calculatedApiName(match.dataset, match.name), "field", reference); + ExpressionCompiler.Binding resolved = fieldPlan.resolve(match.dataset, match.name); + if (!fieldPlan.hasPhysicalDependencies(match.dataset, match.name)) { + throw new IllegalArgumentException("Field reference '" + reference + + "' is a constant row field without a physical dataset anchor; " + + "combine it with a direct field in a derived row expression before using it in a metric"); + } + checkType(resolved.datatype(), getString(targetField, "dataType"), reference); + return resolved; } + Map targetField = exported(targetFields.get(match.dataset), match.name, "field", reference); + String datatype = getString(match.properties, "datatype"); + String targetType = getString(targetField, "dataType"); + if (datatype == null || datatype.isBlank()) datatype = SalesforceDataTypeMapper.toOssie(targetType); + checkType(datatype, targetType, reference); + String datasetName = getString(targetDataset, "apiName"); + return new ExpressionCompiler.Binding(bracket(datasetName) + "." + bracket(getString(targetField, "apiName")), + datatype, datasetName); + } + + private static void checkType(String datatype, String targetType, String reference) { if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { throw new IllegalArgumentException("Field reference '" + reference + "' has no supported datatype; declare a portable field datatype"); } - if (targetType == null || targetType.isBlank() - || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { - throw new IllegalArgumentException("Field reference '" + reference + "' has datatype '" - + datatype + "' but exported Salesforce dataType '" + targetType - + "'; use compatible field types"); + if (targetType == null || targetType.isBlank() || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { + throw new IllegalArgumentException("Field reference '" + reference + "' has datatype '" + datatype + + "' but exported Salesforce dataType '" + targetType + "'; use compatible field types"); } - - String targetDatasetName = getString(targetDataset, "apiName"); - String targetFieldName = getString(targetField, "apiName"); - return new ResolvedField(bracket(targetDatasetName) + "." + bracket(targetFieldName), - datatype, targetDatasetName); } - private static Map exportedItem(List> items, - String name, String kind, String reference) { - List> matches = items.stream() - .filter(item -> name.equals(getString(item, "apiName"))).toList(); - if (matches.isEmpty()) { - throw new IllegalArgumentException("Declared " + kind + " in reference '" + reference - + "' was not exported as a direct Salesforce semantic " + kind - + "; calculated or omitted fields are unsupported in metric references"); - } - if (matches.size() > 1) { - throw new IllegalArgumentException("Ambiguous exported Salesforce " + kind + " for reference '" - + reference + "'; apiName values must be unique"); - } + private static Map exported(Map>> index, + String name, String kind, String reference) { + List> matches = index == null ? List.of() : index.getOrDefault(name, List.of()); + if (matches.isEmpty()) throw new IllegalArgumentException("Declared " + kind + " in reference '" + reference + + "' was not exported as a direct Salesforce semantic " + kind + + "; calculated or omitted fields are unsupported in metric references without a compiled field plan"); + if (matches.size() > 1) throw new IllegalArgumentException("Ambiguous exported Salesforce " + kind + + " for reference '" + reference + "'; apiName values must be unique"); return matches.get(0); } - private static boolean matches(Identifier reference, String declaration, boolean tableau) { - if (declaration == null) { - return false; + private static Map connectedComponents(Set names, List> relationships) { + Map> graph = new LinkedHashMap<>(); + names.forEach(name -> graph.put(name, new HashSet<>())); + for (Map relationship : relationships) { + if (Boolean.FALSE.equals(relationship.get("isEnabled"))) continue; + String left = getString(relationship, "leftSemanticDefinitionApiName"); + String right = getString(relationship, "rightSemanticDefinitionApiName"); + if (graph.containsKey(left) && graph.containsKey(right)) { + graph.get(left).add(right); graph.get(right).add(left); + } } - return tableau ? reference.text().equals(declaration) - : normalize(reference).equals(normalizeDeclaration(declaration)); - } - - private static boolean equivalentDeclaration(String first, String second, boolean tableau) { - return second != null && (tableau ? first.equals(second) - : normalizeDeclaration(first).equals(normalizeDeclaration(second))); + Map result = new HashMap<>(); + for (String name : names) { + if (result.containsKey(name)) continue; + int component = result.size(); + ArrayDeque pending = new ArrayDeque<>(); pending.add(name); + while (!pending.isEmpty()) { + String current = pending.removeFirst(); + if (result.putIfAbsent(current, component) == null) pending.addAll(graph.get(current)); + } + } + return Map.copyOf(result); } - private static String normalize(Identifier identifier) { + static String normalize(Identifier identifier) { return identifier.quoted() ? identifier.text() : identifier.text().toUpperCase(Locale.ROOT); } - private static String normalizeDeclaration(String name) { + static String normalizeDeclaration(String name) { if (name.startsWith("\"") && name.endsWith("\"") && name.length() >= 2) { String text = name.substring(1, name.length() - 1); - String unescaped = text.replace("\"\"", ""); - if (text.isEmpty() || unescaped.contains("\"")) { + if (text.isEmpty() || text.replace("\"\"", "").contains("\"")) { throw new IllegalArgumentException("Invalid quoted declaration name '" + name + "'"); } return text.replace("\"\"", "\""); @@ -217,7 +215,7 @@ private static String normalizeDeclaration(String name) { return name.toUpperCase(Locale.ROOT); } - private static String bracket(String name) { + static String bracket(String name) { if (name == null || name.isBlank() || name.indexOf('[') >= 0 || name.indexOf(']') >= 0 || name.chars().anyMatch(Character::isISOControl)) { throw new IllegalArgumentException("Exported apiName '" + name @@ -226,6 +224,28 @@ private static String bracket(String name) { return "[" + name + "]"; } + private static Map> index(List values, Function name) { + Map> result = new HashMap<>(); + values.forEach(value -> result.computeIfAbsent(name.apply(value), ignored -> new ArrayList<>()).add(value)); + return result; + } + + private static final class NameIndex { + private final Map> exact = new HashMap<>(); + private final Map> sql = new HashMap<>(); + void add(String name, T value) { + if (name == null) return; + exact.computeIfAbsent(name, ignored -> new ArrayList<>()).add(value); + sql.computeIfAbsent(normalizeDeclaration(name), ignored -> new ArrayList<>()).add(value); + } + List find(Identifier identifier, boolean tableau) { + return (tableau ? exact : sql).getOrDefault(tableau ? identifier.text() : normalize(identifier), List.of()); + } + List declarations(String name, boolean tableau) { + return (tableau ? exact : sql).getOrDefault(tableau ? name : normalizeDeclaration(name), List.of()); + } + } + private static List> items(Map map, String key) { List values = getList(map, key); return values == null ? List.of() : streamMaps(values).toList(); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index c258fc34..05c553ae 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -58,20 +58,25 @@ public MetricMappingHandler(ConversionDirection direction, CustomExtensionHandle @Override public void execute(Map sourceData, Map outputData, Map mappings) { + execute(new ConversionContext(sourceData, outputData), mappings); + } + + @Override + public void execute(ConversionContext context, Map mappings) { logger.debug("Mapping metrics in {} direction", direction); if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { - mapOssieToSalesforce(sourceData, outputData, mappings); + mapOssieToSalesforce(context, mappings); } else { - mapSalesforceToOssie(sourceData, outputData, mappings); + mapSalesforceToOssie(context.sourceData(), context.outputData(), mappings); } } /** * Maps Ossie metrics to Salesforce semanticCalculatedMeasurements. */ - private void mapOssieToSalesforce( - Map sourceData, Map outputData, Map mappings) { - + private void mapOssieToSalesforce(ConversionContext context, Map mappings) { + Map sourceData = context.sourceData(); + Map outputData = context.outputData(); List ossieMetrics = getList(sourceData, METRICS); if (ossieMetrics == null) { return; @@ -95,7 +100,7 @@ private void mapOssieToSalesforce( List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); if (sfMetrics != null) { - unwrapExpressions(ossieMetrics, sfMetrics, sourceData, outputData); + unwrapExpressions(ossieMetrics, sfMetrics, context); } else if (!ossieMetrics.isEmpty()) { throw new ConversionException("Metric '" + getString(asMap(ossieMetrics.get(0)), NAME) + "': metric mappings produced no calculated measurements"); @@ -142,20 +147,28 @@ private void mapSalesforceToOssie( * the OSI declarations and the actual emitted fields, including their types. */ private void unwrapExpressions(List ossieMetrics, List sfMetrics, - Map sourceData, Map outputData) { + ConversionContext context) { if (ossieMetrics.size() != sfMetrics.size()) { throw new ConversionException("Metric export count differs from declared metrics: " + streamMaps(ossieMetrics).map(metric -> getString(metric, NAME)).toList()); } + MetricFieldResolver resolver = new MetricFieldResolver(context.sourceData(), context.outputData(), context.fieldPlan()); + MetricCompilationPlan plan = new MetricCompilationPlan(context.sourceData(), resolver); for (int i = 0; i < ossieMetrics.size(); i++) { Map ossieMetric = asMap(ossieMetrics.get(i)); Map sfMetric = asMap(sfMetrics.get(i)); - MetricExpressionTranslator.Result translated = - MetricExpressionTranslator.translate(ossieMetric, sourceData, outputData); + customExtensionHandler.restoreSalesforceCustomExtension(sfMetric, ossieMetric); + ExpressionCompiler.Compiled translated = plan.compile(getString(ossieMetric, NAME)); + String nativeType = getString(sfMetric, DATA_TYPE); + if (nativeType != null && !Set.of("Number", "Currency", "Percentage").contains(nativeType)) { + throw new ConversionException("Metric '" + getString(ossieMetric, NAME) + + "': incompatible Salesforce dataType " + nativeType); + } sfMetric.put(EXPRESSION, translated.expression()); - sfMetric.put(DATA_TYPE, translated.dataType()); + sfMetric.put(DATA_TYPE, nativeType == null ? "Number" : nativeType); sfMetric.put("syntax", "Tua"); sfMetric.put("aggregationType", "UserAgg"); + sfMetric.put("level", "AggregateFunction"); } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java index 20c4949b..f1c30f9b 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java @@ -68,24 +68,18 @@ public void execute(Map sourceData, Map outputDa private void mapOssieToSalesforce( Map sourceData, Map outputData, Map mappings) { + Map relationshipMappings = MappingUtils.filterMappingsByPrefix(mappings, RELATIONSHIPS); + // This handler owns relationships even when there are none. Leaving these entries + // behind lets the final generic handler copy raw Ossie relationships into the output. + relationshipMappings.keySet().forEach(mappings::remove); + List ossieRelationships = getList(sourceData, RELATIONSHIPS); if (ossieRelationships == null) { return; } - - // Validate and filter relationships - remove those with non-existent fields - List validRelationships = validateAndFilterRelationships(ossieRelationships, outputData); - if (validRelationships.isEmpty()) { - return; - } - - // Update sourceData with filtered relationships - sourceData.put(RELATIONSHIPS, validRelationships); - - Map relationshipMappings = MappingUtils.filterMappingsByPrefix(mappings, RELATIONSHIPS); + SalesforceModelValidator.validateRelationshipDeclarations(sourceData, outputData); Map mappedData = GenericMappingEngine.applyMappings(sourceData, relationshipMappings); - relationshipMappings.keySet().forEach(mappings::remove); outputData.putAll(mappedData); @@ -100,6 +94,7 @@ private void mapOssieToSalesforce( if (sfRelationships != null) { applyDefaults(sfRelationships); } + SalesforceModelValidator.validateRelationships(sourceData, outputData); } /** @@ -277,110 +272,15 @@ private void applyDefaults(List sfRelationships) { for (Object relObj : sfRelationships) { Map sfRel = asMap(relObj); - sfRel.putIfAbsent(CARDINALITY, DEFAULT_CARDINALITY); + SalesforceModelValidator.validateRelationshipOptions(sfRel); + // Ossie defines `from` as the many side and `to` as the one side. + // An explicit native value remains an intentional round-trip override. + sfRel.putIfAbsent(CARDINALITY, "ManyToOne"); sfRel.putIfAbsent(IS_ENABLED, true); sfRel.putIfAbsent(JOIN_TYPE, DEFAULT_JOIN_TYPE); } } - /** - * Validates and filters relationships, removing those that reference non-existent fields. (Calculated fields that are not supported) - * - * @param ossieRelationships List of Ossie relationships to validate - * @param outputData The output data containing semanticDataObjects with their fields - * @return Filtered list of valid relationships - */ - private List validateAndFilterRelationships(List ossieRelationships, Map outputData) { - List validRelationships = new ArrayList<>(); - List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); - - for (Object relObj : ossieRelationships) { - Map ossieRel = asMap(relObj); - String relName = getString(ossieRel, NAME); - String fromEntity = getString(ossieRel, FROM); - String toEntity = getString(ossieRel, TO); - - Map fromDataObject = findDataObjectByName(sfDataObjects, fromEntity); - Map toDataObject = findDataObjectByName(sfDataObjects, toEntity); - - if (fromDataObject == null || toDataObject == null) { - logger.debug("Removing relationship '{}' - entity not found", relName); - continue; - } - - List fromColumns = getList(ossieRel, FROM_COLUMNS); - List toColumns = getList(ossieRel, TO_COLUMNS); - - if (!validateColumns(fromColumns, fromDataObject, fromEntity, relName) || - !validateColumns(toColumns, toDataObject, toEntity, relName)) { - continue; - } - validRelationships.add(ossieRel); - } - return validRelationships; - } - - /** - * Validates that all columns exist in the given data object. - * - * @param columns List of column names to validate - * @param dataObject The data object containing the fields - * @param entityName The entity name (for logging) - * @param relName The relationship name (for logging) - * @return true if all columns exist, false otherwise - */ - private boolean validateColumns(List columns, Map dataObject, - String entityName, String relName) { - if (columns == null || columns.isEmpty()) { - return true; - } - - for (Object colObj : columns) { - String columnName = (String) colObj; - if (!fieldExistsInDataObject(dataObject, columnName)) { - logger.debug("Removing relationship '{}' - column '{}' not found in entity '{}'", - relName, columnName, entityName); - return false; - } - } - - return true; - } - - /** - * Finds a data object by its apiName. - */ - private Map findDataObjectByName(List dataObjects, String name) { - for (Object obj : dataObjects) { - Map dataObject = asMap(obj); - String apiName = getString(dataObject, API_NAME); - if (name.equals(apiName)) { - return dataObject; - } - } - return null; - } - - /** - * Checks if a field exists in a data object's semanticDimensions or semanticMeasurements. - */ - private boolean fieldExistsInDataObject(Map dataObject, String fieldName) { - // Check both semanticDimensions and semanticMeasurements - for (String fieldListKey : List.of(SEMANTIC_DIMENSIONS, SEMANTIC_MEASUREMENTS)) { - List fields = getList(dataObject, fieldListKey); - if (fields != null) { - for (Object fieldObj : fields) { - Map field = asMap(fieldObj); - String apiName = getString(field, API_NAME); - if (fieldName.equals(apiName)) { - return true; - } - } - } - } - return false; - } - /** * Checks if a relationship has unsupported field types (Formula or SemanticField). * diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java new file mode 100644 index 00000000..0be0cc68 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.util.DataStructureUtils.*; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.ossie.exception.ConversionException; +import org.apache.ossie.exception.InvalidInputException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Immutable environment bindings. Business expressions and semantic names cannot be overridden. */ +public final class SalesforceBindings { + private record DatasetBinding(String objectName, String objectType, Map fields) {} + private record ModelBinding(String dataspace, Map datasets) {} + private final Map models; + + private SalesforceBindings(Map models) { this.models = Map.copyOf(models); } + + public static SalesforceBindings none() { return new SalesforceBindings(Map.of()); } + + /** Loads JSON or YAML. Unknown properties and duplicate mapping keys are errors. */ + public static SalesforceBindings fromPath(Path path) { + try { return fromString(Files.readString(path)); } + catch (IOException e) { throw new InvalidInputException("Cannot read Salesforce bindings: " + path, e); } + } + + public static SalesforceBindings fromString(String content) { + try { + YAMLFactory factory = new YAMLFactory(); + factory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); + Map root = new ObjectMapper(factory).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .readValue(content, new TypeReference<>() {}); + if (root == null) throw new IllegalArgumentException("bindings document is empty"); + keys(root, Set.of("models"), "bindings"); + Map entries = object(root.get("models"), "bindings.models"); + Map result = new LinkedHashMap<>(); + for (var entry : entries.entrySet()) { + String context = "Bindings for model '" + entry.getKey() + "'"; + Map model = object(entry.getValue(), context); + keys(model, Set.of("dataspace", "datasets"), context); + Map datasets = new LinkedHashMap<>(); + for (var dataset : optionalObject(model, "datasets", context).entrySet()) { + String location = context + ", dataset '" + dataset.getKey() + "'"; + Map definition = object(dataset.getValue(), location); + keys(definition, Set.of("dataObjectName", "dataObjectType", "fields"), location); + Map fields = new LinkedHashMap<>(); + for (var field : optionalObject(definition, "fields", location).entrySet()) { + fields.put(field.getKey(), string(field.getValue(), location + ", field '" + field.getKey() + "'")); + } + datasets.put(dataset.getKey(), new DatasetBinding(optionalString(definition, "dataObjectName", location), + optionalString(definition, "dataObjectType", location), Map.copyOf(fields))); + } + result.put(entry.getKey(), new ModelBinding(optionalString(model, "dataspace", context), Map.copyOf(datasets))); + } + return new SalesforceBindings(result); + } catch (IOException | IllegalArgumentException e) { + throw new InvalidInputException("Invalid Salesforce bindings: " + e.getMessage(), e); + } + } + + boolean isEmpty() { return models.isEmpty(); } + + void validateModels(Set names) { + for (String name : models.keySet()) { + if (!names.contains(name)) throw new ConversionException("Bindings reference unknown model '" + name + "'"); + } + } + + /** Apply only after expressions are bound in their original source scope. */ + void apply(Map source, Map target) { + String modelName = getString(source, "name"); + ModelBinding model = models.get(modelName); + if (model == null) return; + if (model.dataspace != null) target.put("dataspace", model.dataspace); + Map>> sourceDatasets = index(items(source, "datasets"), "name"); + Map>> targetDatasets = index(items(target, "semanticDataObjects"), "apiName"); + for (var entry : model.datasets.entrySet()) { + String datasetName = entry.getKey(); + String context = "Bindings for model '" + modelName + "', dataset '" + datasetName + "'"; + Map sourceDataset = unique(sourceDatasets, datasetName, context); + Map targetDataset = unique(targetDatasets, datasetName, context); + DatasetBinding binding = entry.getValue(); + if (binding.objectName != null) targetDataset.put("dataObjectName", binding.objectName); + if (binding.objectType != null) targetDataset.put("dataObjectType", binding.objectType); + Map>> sourceFields = index(items(sourceDataset, "fields"), "name"); + List> directFields = new java.util.ArrayList<>(items(targetDataset, "semanticDimensions")); + directFields.addAll(items(targetDataset, "semanticMeasurements")); + Map>> targetFields = index(directFields, "apiName"); + for (var field : binding.fields.entrySet()) { + unique(sourceFields, field.getKey(), context + ", field '" + field.getKey() + "'"); + Map targetField = unique(targetFields, field.getKey(), + context + ", field '" + field.getKey() + "' (only direct physical fields can be rebound)"); + targetField.put("dataObjectFieldName", field.getValue()); + } + } + } + + private static Map>> index(List> objects, String key) { + Map>> result = new LinkedHashMap<>(); + for (Map object : objects) { + result.computeIfAbsent(getString(object, key), ignored -> new java.util.ArrayList<>()).add(object); + } + return result; + } + + private static Map unique(Map>> objects, String name, String context) { + List> matches = objects.getOrDefault(name, List.of()); + if (matches.size() != 1) throw new ConversionException(context + ": expected one declared/exported identity, found " + matches.size()); + return matches.get(0); + } + + private static void keys(Map value, Set allowed, String context) { + for (String key : value.keySet()) { + if (!allowed.contains(key)) throw new IllegalArgumentException(context + ": unknown property '" + key + "'"); + } + } + + @SuppressWarnings("unchecked") + private static Map object(Object value, String context) { + if (!(value instanceof Map map)) throw new IllegalArgumentException(context + " must be an object"); + if (map.keySet().stream().anyMatch(key -> !(key instanceof String) || ((String) key).isBlank())) { + throw new IllegalArgumentException(context + " requires nonempty string keys"); + } + return (Map) map; + } + + private static Map optionalObject(Map value, String key, String context) { + return value.containsKey(key) ? object(value.get(key), context + "." + key) : Map.of(); + } + + private static String string(Object value, String context) { + if (!(value instanceof String text) || text.isBlank() || text.chars().anyMatch(Character::isISOControl)) { + throw new IllegalArgumentException(context + " must be a nonempty string without control characters"); + } + return text; + } + + private static String optionalString(Map value, String key, String context) { + return value.containsKey(key) ? string(value.get(key), context + "." + key) : null; + } + + private static List> items(Map value, String key) { + List list = getList(value, key); + return list == null ? List.of() : streamMaps(list).toList(); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java new file mode 100644 index 00000000..a96488fc --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java @@ -0,0 +1,567 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.converter.ConverterConstants.*; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.ossie.exception.ConversionException; + +/** + * Checks references and preservation after all handlers and native extensions have run. + * JSON schema validation remains separate: a structurally valid model can still contain + * duplicate identities, dangling references, omitted entities, or disconnected calculations. + * This validates declared metadata, not actual uniqueness or data in a Salesforce catalog. + */ +public final class SalesforceModelValidator { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Set CARDINALITIES = Set.of( + "OneToOne", "OneToMany", "ManyToOne", "ManyToMany", "Unspecified"); + private static final Set JOIN_TYPES = Set.of("Auto", "Inner", "Left", "Right", "Full"); + private static final List DIRECT_FIELDS = List.of(SEMANTIC_DIMENSIONS, SEMANTIC_MEASUREMENTS); + private static final List CALCULATED_FIELDS = + List.of(SEMANTIC_CALCULATED_DIMENSIONS, SEMANTIC_CALCULATED_MEASUREMENTS); + + /** Validates a model whose source fields are all direct bindings. */ + public void validate(Map sourceModel, Map targetModel) { + validate(sourceModel, targetModel, null); + } + + /** Uses the existing field plan to verify coverage without compiling expressions again. */ + public void validate(Map sourceModel, Map targetModel, + FieldExpressionPlan fieldPlan) { + String modelName = text(sourceModel, NAME, "Ossie model"); + if (!modelName.equals(text(targetModel, API_NAME, "Salesforce model"))) { + fail("Model '" + modelName + "' changed identity during conversion"); + } + Map> sources = index(items(sourceModel, DATASETS), NAME, "dataset", true); + Map> targets = index(items(targetModel, SEMANTIC_DATA_OBJECTS), API_NAME, + "exported dataset", false); + Map> calculations = calculationIndex(targetModel); + Set compiledCalculations = new HashSet<>(); + Map>> targetFields = new LinkedHashMap<>(); + for (var entry : targets.entrySet()) { + String scope = "dataset '" + entry.getKey() + "'"; + text(entry.getValue(), "dataObjectName", scope); + Map> fields = directFields(entry.getValue()); + for (var field : fields.entrySet()) { + text(field.getValue(), DATA_OBJECT_FIELD_NAME, scope + " field '" + field.getKey() + "'"); + } + targetFields.put(entry.getKey(), fields); + } + for (var entry : sources.entrySet()) { + String dataset = entry.getKey(); + require(targets, dataset, "Declared dataset '" + dataset + "' was not exported"); + Map> fields = sourceFields(entry.getValue()); + validateKeys(entry.getValue(), fields, dataset); + for (String field : fields.keySet()) { + if (targetFields.get(dataset).containsKey(field)) { + continue; + } + String calculated = fieldPlan == null ? null : fieldPlan.calculatedApiName(dataset, field); + if (calculated == null || !calculations.containsKey(calculated)) { + fail("Declared field '" + dataset + "." + field + "' was not exported"); + } + compiledCalculations.add(calculated); + } + } + Map> sourceMetrics = index(items(sourceModel, METRICS), NAME, "metric", true); + Map> targetMetrics = index(items(targetModel, SEMANTIC_CALCULATED_MEASUREMENTS), + API_NAME, "exported metric", false); + for (String metric : sourceMetrics.keySet()) { + require(targetMetrics, metric, "Declared metric '" + metric + "' was not exported"); + compiledCalculations.add(metric); + } + validateNativeArrayCoverage(sourceModel, targetModel); + validateRelationshipDeclarations(sourceModel, targetModel); + validateRelationships(sourceModel, targetModel); + validateCalculations(calculations, targetFields, targetModel, compiledCalculations); + } + + private static void validateNativeArrayCoverage(Map source, Map target) { + for (Map extension : items(source, CUSTOM_EXTENSIONS)) { + if (!VENDOR_NAME_VALUE.equals(extension.get(VENDOR_NAME))) continue; + Map nativeModel; + try { + nativeModel = JSON.readValue(text(extension, DATA, "Salesforce extension"), new TypeReference<>() {}); + } catch (java.io.IOException e) { + throw new ConversionException("Model Salesforce extension contains invalid JSON", e); + } + if (nativeModel == null) fail("Model Salesforce extension must contain a JSON object"); + for (String array : List.of(SEMANTIC_DATA_OBJECTS, SEMANTIC_RELATIONSHIPS, + SEMANTIC_CALCULATED_DIMENSIONS, SEMANTIC_CALCULATED_MEASUREMENTS)) { + Map> expected = index(items(nativeModel, array), API_NAME, + "native extension " + array, false); + Map> actual = index(items(target, array), API_NAME, array, false); + for (String name : expected.keySet()) { + require(actual, name, "Native extension " + array + " entity '" + name + + "' was not exported; merge it into the core model instead of silently replacing its array"); + } + } + } + } + + /** Checks every declared edge before mapping; no field or edge may be silently dropped. */ + static void validateRelationshipDeclarations(Map source, Map target) { + Map> sources = index(items(source, DATASETS), NAME, "dataset", true); + Map> targets = index(items(target, SEMANTIC_DATA_OBJECTS), API_NAME, + "exported dataset", false); + Map> relationships = index(items(source, RELATIONSHIPS), NAME, + "relationship", true); + for (var entry : relationships.entrySet()) { + String scope = "Relationship '" + entry.getKey() + "'"; + Map relation = entry.getValue(); + String from = text(relation, FROM, scope); + String to = text(relation, TO, scope); + Map fromDataset = require(sources, from, scope + " has unknown from dataset '" + from + "'"); + Map toDataset = require(sources, to, scope + " has unknown to dataset '" + to + "'"); + Map fromTarget = require(targets, from, scope + " has unexported dataset '" + from + "'"); + Map toTarget = require(targets, to, scope + " has unexported dataset '" + to + "'"); + List fromColumns = names(relation.get(FROM_COLUMNS), scope + " from_columns"); + List toColumns = names(relation.get(TO_COLUMNS), scope + " to_columns"); + if (fromColumns.size() != toColumns.size()) { + fail(scope + " must have the same number of from_columns and to_columns"); + } + checkColumns(scope, from, fromColumns, fromDataset, fromTarget); + checkColumns(scope, to, toColumns, toDataset, toTarget); + validateKeys(toDataset, sourceFields(toDataset), to); + validateKeys(fromDataset, sourceFields(fromDataset), from); + } + } + + /** Validates final native edges, including relationships restored from native extensions. */ + static void validateRelationships(Map source, Map target) { + Map> sourceDatasets = index(items(source, DATASETS), NAME, "dataset", true); + Map> datasets = index(items(target, SEMANTIC_DATA_OBJECTS), API_NAME, + "exported dataset", false); + Map> relationships = index(items(target, SEMANTIC_RELATIONSHIPS), API_NAME, + "exported relationship", false); + for (var entry : relationships.entrySet()) { + String scope = "Relationship '" + entry.getKey() + "'"; + Map relation = entry.getValue(); + String left = text(relation, LEFT_SEMANTIC_DEFINITION_API_NAME, scope); + String right = text(relation, RIGHT_SEMANTIC_DEFINITION_API_NAME, scope); + Map leftDataset = require(datasets, left, scope + " has unknown endpoint '" + left + "'"); + Map rightDataset = require(datasets, right, scope + " has unknown endpoint '" + right + "'"); + List> criteria = items(relation, CRITERIA); + if (criteria.isEmpty()) fail(scope + " requires nonempty criteria"); + for (Map criterion : criteria) { + validateCriterion(scope, criterion, LEFT_FIELD_TYPE, LEFT_SEMANTIC_FIELD_API_NAME, left, leftDataset); + validateCriterion(scope, criterion, RIGHT_FIELD_TYPE, RIGHT_SEMANTIC_FIELD_API_NAME, right, rightDataset); + } + validateRelationshipOptions(relation); + } + for (Map relation : items(source, RELATIONSHIPS)) { + String name = text(relation, NAME, "Ossie relationship"); + Map exported = require(relationships, name, + "Declared relationship '" + name + "' was not exported"); + String scope = "Relationship '" + name + "'"; + if (!text(relation, FROM, scope).equals(exported.get(LEFT_SEMANTIC_DEFINITION_API_NAME)) + || !text(relation, TO, scope).equals(exported.get(RIGHT_SEMANTIC_DEFINITION_API_NAME))) { + fail(scope + " changed endpoints during conversion"); + } + List from = names(relation.get(FROM_COLUMNS), scope + " from_columns"); + List to = names(relation.get(TO_COLUMNS), scope + " to_columns"); + List> criteria = items(exported, CRITERIA); + if (from.size() != to.size() || criteria.size() != from.size()) fail(scope + " changed composite key arity"); + for (int i = 0; i < from.size(); i++) { + if (!from.get(i).equals(criteria.get(i).get(LEFT_SEMANTIC_FIELD_API_NAME)) + || !to.get(i).equals(criteria.get(i).get(RIGHT_SEMANTIC_FIELD_API_NAME))) { + fail(scope + " changed join key correspondence during conversion"); + } + } + // Native round-trip metadata can explicitly reverse or relax core cardinality. + // Check keys on the side(s) that the final native edge actually declares unique. + String cardinality = (String) exported.get(CARDINALITY); + if ("ManyToOne".equals(cardinality) || "OneToOne".equals(cardinality)) { + validateUniqueSide(scope, "to_columns", to, sourceDatasets.get(relation.get(TO))); + } + if ("OneToMany".equals(cardinality) || "OneToOne".equals(cardinality)) { + validateUniqueSide(scope, "from_columns", from, sourceDatasets.get(relation.get(FROM))); + } + } + } + + static void validateRelationshipOptions(Map relation) { + String scope = "Relationship '" + relation.get(API_NAME) + "'"; + if (relation.containsKey(CARDINALITY)) { + String cardinality = text(relation, CARDINALITY, scope); + if (!CARDINALITIES.contains(cardinality)) fail(scope + " has invalid cardinality '" + cardinality + "'"); + } + if (relation.containsKey(JOIN_TYPE)) { + String joinType = text(relation, JOIN_TYPE, scope); + if (!JOIN_TYPES.contains(joinType)) fail(scope + " has invalid joinType '" + joinType + "'"); + } + if (relation.containsKey(IS_ENABLED) && !(relation.get(IS_ENABLED) instanceof Boolean)) { + fail(scope + " requires a Boolean isEnabled when supplied"); + } + } + + private static void validateUniqueSide(String scope, String side, List columns, Map dataset) { + if (dataset == null) fail(scope + " references an undeclared dataset"); + String name = (String) dataset.get(NAME); + List> keys = validateKeys(dataset, sourceFields(dataset), name); + if (!keys.isEmpty() && keys.stream().noneMatch(columns::containsAll)) { + fail(scope + " " + side + " do not include a declared primary_key or unique_key of '" + name + "'"); + } + } + + private static void validateCriterion(String scope, Map criterion, String typeKey, + String fieldKey, String dataset, Map target) { + Object type = criterion.get(typeKey); + if (type != null && !FIELD_TYPE_TABLE_FIELD.equals(type)) { + fail(scope + " uses unsupported calculated join key type '" + type + "'; only direct table fields are supported"); + } + String field = text(criterion, fieldKey, scope + " criterion"); + require(directFields(target), field, scope + " references missing direct field '" + dataset + "." + field + "'"); + } + + private static void checkColumns(String scope, String dataset, List columns, + Map source, Map target) { + Map> declared = sourceFields(source); + Map> exported = directFields(target); + for (String column : columns) { + require(declared, column, scope + " references undeclared field '" + dataset + "." + column + "'"); + require(exported, column, scope + " field '" + dataset + "." + column + + "' was not exported as a direct field; calculated join keys are unsupported"); + } + } + + private static List> validateKeys(Map dataset, + Map> fields, String name) { + List> keys = new ArrayList<>(); + if (dataset.containsKey("primary_key")) { + keys.add(new LinkedHashSet<>(names(dataset.get("primary_key"), "Dataset '" + name + "' primary_key"))); + } + Object unique = dataset.get("unique_keys"); + if (unique != null) { + if (!(unique instanceof List)) fail("Dataset '" + name + "' unique_keys must be an array"); + for (Object key : (List) unique) { + keys.add(new LinkedHashSet<>(names(key, "Dataset '" + name + "' unique_key"))); + } + } + for (Set key : keys) { + for (String field : key) require(fields, field, "Dataset '" + name + "' key references unknown field '" + field + "'"); + } + // The native schema has no primary/unique-key constraint. primaryNameField is + // a display identifier, and keyQualifierName is not a uniqueness declaration. + return keys; + } + + private static void validateCalculations(Map> calculations, + Map>> fields, Map target, + Set compiledCalculations) { + Map> dependencies = new LinkedHashMap<>(); + Map> datasets = new LinkedHashMap<>(); + for (var entry : calculations.entrySet()) { + String scope = "Calculated field '" + entry.getKey() + "'"; + if (!"Tua".equals(entry.getValue().get("syntax"))) fail(scope + " requires syntax 'Tua'"); + String expression = text(entry.getValue(), EXPRESSION, scope); + Set refs = new LinkedHashSet<>(); + Set sources = new LinkedHashSet<>(); + for (List reference : references(expression, scope)) { + if (reference.size() == 2) { + String dataset = reference.get(0); + Map> table = require(fields, dataset, + scope + " references unknown dataset '" + dataset + "'"); + require(table, reference.get(1), scope + " references missing field '" + + dataset + "." + reference.get(1) + "'"); + sources.add(dataset); + } else { + String name = reference.get(0); + require(calculations, name, scope + " references unknown calculated field '" + name + "'"); + refs.add(name); + } + } + for (Map dependency : items(entry.getValue(), DEPENDENCIES)) { + String definition = text(dependency, DEPENDENT_DEFINITION_API_NAME, scope + " dependency"); + Object field = dependency.get("dependentFieldApiName"); + if (field != null) { + Map> table = require(fields, definition, + scope + " dependency references unknown dataset '" + definition + "'"); + String name = text(dependency, "dependentFieldApiName", scope + " dependency"); + require(table, name, scope + " dependency references missing field '" + definition + "." + name + "'"); + } else if (!fields.containsKey(definition) && !calculations.containsKey(definition)) { + fail(scope + " dependency references unknown definition '" + definition + "'"); + } + } + dependencies.put(entry.getKey(), refs); + datasets.put(entry.getKey(), sources); + } + List order = collectDatasets(dependencies, datasets); + validateNativeCalculations(order, calculations, fields, datasets, target, compiledCalculations); + Map> graph = new LinkedHashMap<>(); + fields.keySet().forEach(name -> graph.put(name, new LinkedHashSet<>())); + for (Map relation : items(target, SEMANTIC_RELATIONSHIPS)) { + // Optional native metadata may be absent. Only explicitly enabled edges + // establish connectivity; absence is not proof of the target's default. + if (!Boolean.TRUE.equals(relation.get(IS_ENABLED))) continue; + String left = (String) relation.get(LEFT_SEMANTIC_DEFINITION_API_NAME); + String right = (String) relation.get(RIGHT_SEMANTIC_DEFINITION_API_NAME); + graph.get(left).add(right); + graph.get(right).add(left); + } + for (var entry : datasets.entrySet()) { + if (entry.getValue().size() < 2) continue; + Set reached = new HashSet<>(); + List pending = new ArrayList<>(List.of(entry.getValue().iterator().next())); + for (int i = 0; i < pending.size(); i++) { + String dataset = pending.get(i); + if (reached.add(dataset)) pending.addAll(graph.get(dataset)); + } + if (!reached.containsAll(entry.getValue())) { + fail("Calculated field '" + entry.getKey() + "' references datasets disconnected by enabled relationships"); + } + } + } + + private static List collectDatasets(Map> dependencies, Map> datasets) { + Map remaining = new LinkedHashMap<>(); + Map> consumers = new HashMap<>(); + ArrayDeque ready = new ArrayDeque<>(); + for (var entry : dependencies.entrySet()) { + remaining.put(entry.getKey(), entry.getValue().size()); + if (entry.getValue().isEmpty()) ready.add(entry.getKey()); + for (String dependency : entry.getValue()) { + consumers.computeIfAbsent(dependency, ignored -> new ArrayList<>()).add(entry.getKey()); + } + } + int processed = 0; + List order = new ArrayList<>(); + while (!ready.isEmpty()) { + String name = ready.removeFirst(); + order.add(name); + processed++; + for (String consumer : consumers.getOrDefault(name, List.of())) { + datasets.get(consumer).addAll(datasets.get(name)); + if (remaining.compute(consumer, (ignored, count) -> count - 1) == 0) ready.add(consumer); + } + } + if (processed != dependencies.size()) { + String cyclic = remaining.entrySet().stream().filter(entry -> entry.getValue() > 0) + .map(Map.Entry::getKey).findFirst().orElseThrow(); + fail("Cyclic calculated field reference involving '" + cyclic + "'"); + } + return order; + } + + private static void validateNativeCalculations(List order, + Map> calculations, + Map>> fields, Map> datasets, + Map target, Set alreadyCompiled) { + Set measurements = index(items(target, SEMANTIC_CALCULATED_MEASUREMENTS), API_NAME, + "calculated measurement", false).keySet(); + Map verified = new HashMap<>(); + for (String name : order) { + if (alreadyCompiled.contains(name)) continue; + Map calculation = calculations.get(name); + String scope = "Native calculated field '" + name + "'"; + try { + ExpressionCompiler.Compiled compiled = ExpressionCompiler.compile( + ExpressionCompiler.parse(text(calculation, EXPRESSION, scope), DIALECT_TABLEAU), reference -> { + List parts = reference.parts(); + if (parts.size() == 2) { + String dataset = parts.get(0).text(); + String field = parts.get(1).text(); + Map> table = require(fields, dataset, + scope + " references unknown dataset '" + dataset + "'"); + Map item = require(table, field, scope + " references missing field '" + field + "'"); + return new ExpressionCompiler.Binding("[" + dataset + "].[" + field + "]", + nativeDatatype(item, scope + " reference '" + dataset + "." + field + "'"), + dataset, ExpressionCompiler.Level.ROW); + } + if (parts.size() != 1) throw new IllegalArgumentException("reference must be dataset.field or calculated field"); + String dependency = parts.get(0).text(); + Map item = require(calculations, dependency, + scope + " references unknown calculated field '" + dependency + "'"); + ExpressionCompiler.Compiled checked = verified.get(dependency); + String datatype = checked == null ? nativeDatatype(item, scope + " reference '" + dependency + "'") + : checked.datatype(); + ExpressionCompiler.Level level = checked == null + ? measurements.contains(dependency) ? ExpressionCompiler.Level.AGGREGATE : ExpressionCompiler.Level.ROW + : checked.level(); + return new ExpressionCompiler.Binding("[" + dependency + "]", datatype, datasets.get(dependency), level); + }); + String type = calculation.get(DATA_TYPE) instanceof String value ? value : null; + String datatype = compiled.datatype() == null ? SalesforceDataTypeMapper.toOssie(type) : compiled.datatype(); + if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { + throw new IllegalArgumentException("expression needs a supported native dataType to determine its result type"); + } + boolean measurement = measurements.contains(name); + if (measurement && !Set.of("Integer", "Decimal", "Float").contains(datatype)) { + throw new IllegalArgumentException("calculated measurements must return a number"); + } + String expectedLevel = calculation.containsKey("level") ? text(calculation, "level", scope) + : measurement ? "AggregateFunction" : "Row"; + if (!Set.of("Row", "AggregateFunction").contains(expectedLevel)) { + throw new IllegalArgumentException("unsupported native level '" + expectedLevel + "'"); + } + if (compiled.level() == ExpressionCompiler.Level.AGGREGATE && expectedLevel.equals("Row") + || compiled.level() == ExpressionCompiler.Level.ROW && expectedLevel.equals("AggregateFunction")) { + throw new IllegalArgumentException("expression aggregation conflicts with native level '" + expectedLevel + "'"); + } + if (type != null && !SalesforceDataTypeMapper.areCompatible(datatype, type)) { + throw new IllegalArgumentException("expression datatype '" + datatype + + "' conflicts with native dataType '" + type + "'"); + } + verified.put(name, new ExpressionCompiler.Compiled(compiled.expression(), datatype, + compiled.level(), compiled.datasets())); + } catch (IllegalArgumentException e) { + throw new ConversionException(scope + ": " + e.getMessage(), e); + } + } + } + + private static String nativeDatatype(Map item, String scope) { + String type = SalesforceDataTypeMapper.toOssie(item.get(DATA_TYPE) instanceof String value ? value : null); + if (SalesforceDataTypeMapper.toSalesforce(type) == null) fail(scope + " needs a supported dataType"); + return type; + } + + // A reference scanner, not a second formula parser. The compiler owns grammar and + // type validation; this checks final names after native extensions have been restored. + private static List> references(String expression, String scope) { + List> references = new ArrayList<>(); + for (int i = 0; i < expression.length();) { + char c = expression.charAt(i); + if (c == '\'' || c == '"') { + char quote = c; + i++; + boolean closed = false; + while (i < expression.length()) { + char next = expression.charAt(i++); + if (next == '\\' && i < expression.length()) { i++; continue; } + if (next == quote) { + if (i < expression.length() && expression.charAt(i) == quote) { i++; continue; } + closed = true; + break; + } + } + if (!closed) fail(scope + " has an unterminated string literal"); + } else if (c == '[') { + List parts = new ArrayList<>(); + do { + int end = expression.indexOf(']', i + 1); + if (end < 0 || end == i + 1) fail(scope + " has an invalid bracket reference"); + parts.add(expression.substring(i + 1, end)); + i = end + 1; + while (i < expression.length() && Character.isWhitespace(expression.charAt(i))) i++; + if (i >= expression.length() || expression.charAt(i) != '.') break; + i++; + while (i < expression.length() && Character.isWhitespace(expression.charAt(i))) i++; + if (i >= expression.length() || expression.charAt(i) != '[') fail(scope + " has an invalid qualified reference"); + } while (true); + if (parts.size() > 2) fail(scope + " references more than dataset.field"); + references.add(parts); + } else { + i++; + } + } + return references; + } + + private static Map> calculationIndex(Map model) { + List> calculations = new ArrayList<>(); + for (String key : CALCULATED_FIELDS) calculations.addAll(items(model, key)); + return index(calculations, API_NAME, "model-level calculated field", false); + } + + private static Map> sourceFields(Map dataset) { + return index(items(dataset, FIELDS), NAME, "field in dataset '" + dataset.get(NAME) + "'", true); + } + + private static Map> directFields(Map dataset) { + List> fields = new ArrayList<>(); + for (String key : DIRECT_FIELDS) fields.addAll(items(dataset, key)); + return index(fields, API_NAME, "field in exported dataset '" + dataset.get(API_NAME) + "'", false); + } + + private static Map> index(List> values, + String key, String kind, boolean normalize) { + Map> result = new LinkedHashMap<>(); + Set identities = new HashSet<>(); + for (Map value : values) { + String name = text(value, key, kind); + String identity = normalize ? normalize(name) : name; + if (!identities.add(identity)) fail("Duplicate " + kind + " identity '" + name + "'"); + result.put(name, value); + } + return result; + } + + private static String normalize(String name) { + return name.length() >= 2 && name.startsWith("\"") && name.endsWith("\"") + ? name.substring(1, name.length() - 1).replace("\"\"", "\"") : name.toUpperCase(Locale.ROOT); + } + + private static List names(Object value, String scope) { + if (!(value instanceof List values) || values.isEmpty()) fail(scope + " must be a nonempty array"); + List result = new ArrayList<>(); + Set unique = new HashSet<>(); + for (Object item : (List) value) { + if (!(item instanceof String name) || name.isBlank()) fail(scope + " contains a blank or non-string field name"); + String name = (String) item; + if (!unique.add(name)) fail(scope + " contains duplicate field '" + name + "'"); + result.add(name); + } + return result; + } + + @SuppressWarnings("unchecked") + private static List> items(Map object, String key) { + Object value = object.get(key); + if (value == null) return List.of(); + if (!(value instanceof List)) fail("Property '" + key + "' must be an array"); + List> result = new ArrayList<>(); + for (Object item : (List) value) { + if (!(item instanceof Map)) fail("Property '" + key + "' must contain objects"); + result.add((Map) item); + } + return result; + } + + private static String text(Map object, String key, String scope) { + Object value = object.get(key); + if (!(value instanceof String text) || text.isBlank()) fail(scope + " requires a nonempty '" + key + "'"); + return (String) value; + } + + private static T require(Map values, String key, String error) { + T value = values.get(key); + if (value == null) fail(error); + return value; + } + + private static void fail(String message) { + throw new ConversionException(message); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java new file mode 100644 index 00000000..73838d16 --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.converter.ExpressionAst.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import net.sf.jsqlparser.expression.*; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; +import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.AllColumns; + +/** Adapts a completely consumed JSqlParser expression into the explicitly supported compiler AST. */ +final class SqlExpressionParser { + private final String dialect; + private int depth; + private SqlExpressionParser(String dialect) { this.dialect = dialect; } + + static Node parse(String text, String dialect) { + try { + Expression expression = CCJSqlParserUtil.parseCondExpression(text, false, + parser -> parser.withSquareBracketQuotation(dialect.equals("ANSI_SQL"))); + if (expression == null) throw new IllegalArgumentException("could not parse a complete SQL expression"); + return new SqlExpressionParser(dialect).adapt(expression); + } catch (net.sf.jsqlparser.JSQLParserException e) { + throw new IllegalArgumentException(dialect + " expression has unsupported or unexpected token: " + + e.getMessage(), e); + } catch (StackOverflowError e) { + throw new IllegalArgumentException("expression nesting exceeds parser limits", e); + } + } + + private Node adapt(Expression expression) { + if (++depth > 128) throw new IllegalArgumentException("expression nesting exceeds 128 levels"); + try { return adaptNode(expression); } + finally { depth--; } + } + + private Node adaptNode(Expression expression) { + if (expression instanceof ParenthesedExpressionList list && list.size() == 1) { + return adapt(list.get(0)); + } + if (expression instanceof LongValue || expression instanceof DoubleValue) { + return new Literal(ExpressionTokens.number(expression.toString())); + } + if (expression instanceof NullValue) return new Literal(null); + if (expression instanceof BooleanValue value) return new Literal(value.getValue()); + if (expression instanceof StringValue value) { + if (value.getPrefix() != null) throw unsupported(expression); + return new Literal(value.getValue().replace("''", "'")); + } + if (expression instanceof Column column) return column(column); + if (expression instanceof SignedExpression signed) { + if (signed.getSign() != '+' && signed.getSign() != '-') throw unsupported(expression); + return new Unary(String.valueOf(signed.getSign()), adapt(signed.getExpression())); + } + if (expression instanceof NotExpression not) { + if (not.isExclamationMark()) throw unsupported(expression); + return new Unary("NOT", adapt(not.getExpression())); + } + if (expression instanceof IsNullExpression test) { + if (test.isUseIsNull() || test.isUseNotNull()) throw unsupported(expression); + Node result = new Unary("ISNULL", adapt(test.getLeftExpression())); + return test.isNot() ? new Unary("NOT", result) : result; + } + if (expression instanceof BinaryExpression binary) { + if (binary instanceof net.sf.jsqlparser.expression.operators.relational.SupportsOldOracleJoinSyntax oracle + && (oracle.getOldOracleJoinSyntax() != 0 || oracle.getOraclePriorPosition() != 0)) { + throw unsupported(expression); + } + String operator = binary.getStringExpression().toUpperCase(Locale.ROOT); + if (!Set.of("+", "-", "*", "/", "AND", "OR", "=", "!=", "<>", "<", ">", "<=", ">=").contains(operator)) { + throw unsupported(expression); + } + return new Binary(operator.equals("<>") ? "!=" : operator, + adapt(binary.getLeftExpression()), adapt(binary.getRightExpression())); + } + if (expression instanceof Function function) return function(function); + if (expression instanceof CaseExpression conditional) { + List branches = new ArrayList<>(); + // The original bounded contract supports searched CASE. Simple CASE can be + // added with explicit type checking and evaluation-count guarantees later. + if (conditional.getSwitchExpression() != null) throw unsupported(expression); + for (WhenClause branch : conditional.getWhenClauses()) { + branches.add(adapt(branch.getWhenExpression())); + branches.add(adapt(branch.getThenExpression())); + } + return new Conditional(branches, conditional.getElseExpression() == null + ? new Literal(null) : adapt(conditional.getElseExpression())); + } + if (expression instanceof AllColumns) { + throw new IllegalArgumentException("COUNT(*) is unsupported; name a declared field to count"); + } + throw unsupported(expression); + } + + private Node function(Function function) { + String name = function.getName(); + if (name == null || !name.matches("[A-Za-z_][A-Za-z_0-9]*")) { + throw new IllegalArgumentException("quoted or qualified function names are unsupported"); + } + boolean positionSyntax = name.equalsIgnoreCase("POSITION") && function.getNamedParameters() != null; + if (function.isUnique() || function.isEscaped() || function.getNamedParameters() != null && !positionSyntax + || function.getAttribute() != null || function.getKeep() != null + || function.getNullHandling() != null || function.isIgnoreNullsOutside() + || function.isIgnoreNulls() || function.getLimit() != null + || function.getHavingClause() != null || function.getExtraKeyword() != null + || function.getOnOverflowTruncate() != null + || function.getOrderByElements() != null && !function.getOrderByElements().isEmpty()) { + throw new IllegalArgumentException("unsupported function modifiers for " + name); + } + if (function.isAllColumns()) { + throw new IllegalArgumentException("explicit ALL function modifier is outside the supported SQL subset"); + } + List arguments = new ArrayList<>(); + if (positionSyntax) { + var named = function.getNamedParameters(); + if (named.size() != 2 || named.getNames().size() != 2 + || !"IN".equalsIgnoreCase(named.getNames().get(1))) { + throw new IllegalArgumentException("unsupported POSITION argument syntax"); + } + for (Expression argument : named) arguments.add(adapt(argument)); + } + if (function.getParameters() != null) { + for (Expression argument : function.getParameters()) arguments.add(adapt(argument)); + } + return new Call(name.toUpperCase(Locale.ROOT), arguments, function.isDistinct()); + } + + private Node column(Column column) { + if (column.getArrayConstructor() != null) throw unsupported(column); + // JSqlParser 5.3's Table accessors split a quoted name containing a dot + // ("Order.Items") into schema/table parts. Retain the original token + // boundaries instead of binding that expression to a different object. + List raw = new ArrayList<>(); + var source = column.getASTNode(); + if (source == null) throw new IllegalArgumentException("field reference has no source identifier tokens"); + boolean identifier = true; + for (var token = source.jjtGetFirstToken(); token != null; token = token.next) { + if (identifier) raw.add(token.image); + else if (!token.image.equals(".")) throw unsupported(column); + identifier = !identifier; + if (token == source.jjtGetLastToken()) break; + } + if (raw.isEmpty() || identifier) throw unsupported(column); + boolean bracket = raw.get(0).startsWith("["); + List parts = new ArrayList<>(); + for (String part : raw) { + if (bracket != part.startsWith("[")) { + throw new IllegalArgumentException("do not mix bracketed and SQL field identifiers"); + } + boolean quoted = part.startsWith("\"") || part.startsWith("["); + if (quoted) { + String end = bracket ? "]" : "\""; + part = part.substring(1, part.length() - 1).replace(end + end, end); + } + parts.add(new MetricFieldResolver.Identifier(part, quoted)); + } + return new Field(new ExpressionCompiler.Reference(parts, bracket)); + } + + private IllegalArgumentException unsupported(Expression expression) { + return new IllegalArgumentException(dialect + " unsupported SQL expression " + + expression.getClass().getSimpleName() + + "; only documented expression capabilities can be converted"); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java new file mode 100644 index 00000000..a50efcfa --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.converter.ExpressionAst.*; +import java.math.BigDecimal; +import java.util.List; + +/** Emits only checked AST nodes into the target grammar, with bounded expansion. */ +final class TuaExpressionEmitter { + private static final int MAX_OUTPUT = 131072; + private final StringBuilder output = new StringBuilder(); + String emit(Typed expression) { append(expression); return output.toString(); } + private void text(String text) { + if ((long) output.length() + text.length() > MAX_OUTPUT) { + throw new IllegalArgumentException("translated expression exceeds 131072 characters"); + } + output.append(text); + } + private void append(Typed value) { + Node node = value.node(); List children = value.children(); + if (node instanceof Literal literal) { + Object content = literal.value(); + text(content == null ? "NULL" : content instanceof String string ? "'" + string.replace("'", "''") + "'" + : content instanceof Boolean bool ? bool ? "TRUE" : "FALSE" : ((BigDecimal) content).toPlainString()); + } else if (node instanceof Field) { + text(value.binding().expression()); + } else if (node instanceof Unary unary) { + switch (unary.operator()) { + case "+" -> append(children.get(0)); + case "-" -> { text("(-"); append(children.get(0)); text(")"); } + case "NOT" -> { text("(NOT "); append(children.get(0)); text(")"); } + case "ISNULL" -> { text("ISNULL("); append(children.get(0)); text(")"); } + default -> throw new IllegalStateException("unvalidated unary operator"); + } + } else if (node instanceof Binary binary) { + text("("); append(children.get(0)); text(" " + binary.operator() + " "); append(children.get(1)); text(")"); + } else if (node instanceof Conditional) { + text("(IF "); + for (int i = 0; i < children.size() - 1; i += 2) { + if (i > 0) text(" ELSEIF "); + append(children.get(i)); text(" THEN "); append(children.get(i + 1)); + } + text(" ELSE "); append(children.get(children.size() - 1)); text(" END)"); + } else { + Call call = (Call) node; + ExpressionFunctionRegistry.Spec spec = ExpressionFunctionRegistry.get(call.name()); + if (spec.rule() == ExpressionFunctionRegistry.Rule.COALESCE) { + for (int i = 0; i < children.size() - 1; i++) { text("IFNULL("); append(children.get(i)); text(", "); } + append(children.get(children.size() - 1)); + for (int i = 0; i < children.size() - 1; i++) text(")"); + } else if (spec.rule() == ExpressionFunctionRegistry.Rule.NULLIF) { + text("(IF ("); append(children.get(0)); text(" = "); append(children.get(1)); + text(") THEN NULL ELSE "); append(children.get(0)); text(" END)"); + } else { + text((call.distinct() ? "COUNTD" : spec.target()) + "("); + if (call.name().equals("POSITION")) { + append(children.get(1)); text(", "); append(children.get(0)); + } else { + for (int i = 0; i < children.size(); i++) { if (i > 0) text(", "); append(children.get(i)); } + } + text(")"); + } + } + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java new file mode 100644 index 00000000..004d08ff --- /dev/null +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.apache.ossie.converter.ExpressionAst.*; +import static org.apache.ossie.converter.ExpressionTokens.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** Native Tua frontend for the bounded supported grammar; produces the same AST as SQL. */ +final class TuaExpressionParser { + private final List tokens; + private int position; + private int depth; + TuaExpressionParser(List tokens) { this.tokens = tokens; } + + Node parse() { + Node node = expression(); + if (peek().kind() != Kind.END) throw error("unsupported or unexpected token '" + peek().text() + "'"); + return node; + } + private Node expression() { + if (++depth > 128) throw error("expression nesting exceeds 128 levels"); + try { return or(); } finally { depth--; } + } + private Node or() { + Node value = and(); + while (take("OR")) value = new Binary("OR", value, and()); + return value; + } + private Node and() { + Node value = not(); + while (take("AND")) value = new Binary("AND", value, not()); + return value; + } + private Node not() { + int count = 0; + while (take("NOT")) if (++count > 128) throw error("too many unary operators"); + Node value = comparison(); + while (count-- > 0) value = new Unary("NOT", value); + return value; + } + private Node comparison() { + Node value = additive(); + if (at("IS")) throw error("use ISNULL in TABLEAU expressions"); + if (Set.of("=", "!=", "<>", "<", "<=", ">", ">=").contains(peek().text())) { + String op = next().text(); + return new Binary(op.equals("<>") ? "!=" : op, value, additive()); + } + return value; + } + private Node additive() { + Node value = multiplicative(); + while (at("+") || at("-")) value = new Binary(next().text(), value, multiplicative()); + return value; + } + private Node multiplicative() { + Node value = unary(); + while (at("*") || at("/")) value = new Binary(next().text(), value, unary()); + return value; + } + private Node unary() { + List signs = new ArrayList<>(); + while (at("+") || at("-")) { + if (signs.size() >= 128) throw error("too many unary operators"); + signs.add(next().text()); + } + Node value = primary(); + for (int i = signs.size() - 1; i >= 0; i--) value = new Unary(signs.get(i), value); + return value; + } + private Node primary() { + if (take("(")) { Node value = expression(); expect(")"); return value; } + if (at("CASE")) throw error("searched CASE is SQL; use IF in TABLEAU"); + if (take("IF")) return conditional(); + if (take("NULL")) return new Literal(null); + if (at("TRUE") || at("FALSE")) return new Literal(Boolean.valueOf(next().text())); + Token token = next(); + if (token.kind() == Kind.NUMBER) return new Literal(ExpressionTokens.number(token.text())); + if (token.kind() == Kind.STRING) return new Literal(token.text()); + if (token.kind() != Kind.WORD && token.kind() != Kind.IDENTIFIER) { + throw error("expected a value, found '" + token.text() + "'"); + } + if (take("(")) { + if (token.kind() != Kind.WORD) throw error("quoted function names are unsupported"); + return function(token.text().toUpperCase(Locale.ROOT)); + } + List parts = new ArrayList<>(); + addIdentifier(parts, token); + while (take(".")) addIdentifier(parts, next()); + // One bracketed name is a semantic metric reference; the model resolver + // distinguishes it from an unknown or ambiguous field. Physical fields + // still require dataset qualification in the field resolver. + if (parts.size() > 2) throw error("TABLEAU references must use [metric] or [dataset].[field] notation"); + return new Field(new ExpressionCompiler.Reference(parts, true)); + } + private void addIdentifier(List parts, Token token) { + if (token.kind() != Kind.IDENTIFIER || !token.bracket()) { + throw error("TABLEAU fields must use [dataset].[field] notation"); + } + parts.add(new MetricFieldResolver.Identifier(token.text(), true)); + } + private Node conditional() { + List branches = new ArrayList<>(); + do { + branches.add(expression()); expect("THEN"); branches.add(expression()); + } while (take("ELSEIF")); + Node otherwise = take("ELSE") ? expression() : new Literal(null); + expect("END"); + return new Conditional(branches, otherwise); + } + private Node function(String name) { + boolean distinct = take("DISTINCT"); + if (at("*")) throw error("COUNT(*) is unsupported; name a declared field to count"); + List arguments = new ArrayList<>(); + if (!at(")")) do { arguments.add(expression()); } while (take(",")); + expect(")"); + return new Call(name, arguments, distinct); + } + private Token peek() { return tokens.get(position); } + private Token next() { Token token = peek(); if (token.kind() != Kind.END) position++; return token; } + private boolean at(String text) { + return (peek().kind() == Kind.WORD || peek().kind() == Kind.SYMBOL) && peek().text().equalsIgnoreCase(text); + } + private boolean take(String text) { if (!at(text)) return false; next(); return true; } + private void expect(String text) { if (!take(text)) throw error("expected " + text + ", found '" + peek().text() + "'"); } + private IllegalArgumentException error(String message) { + return new IllegalArgumentException("TABLEAU at character " + (peek().offset() + 1) + ": " + message); + } +} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java index 95fef55c..abac7ddf 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java @@ -20,6 +20,7 @@ package org.apache.ossie.converter.pipeline; import java.util.Map; +import org.apache.ossie.converter.ConversionContext; /** * Base interface for pipeline steps. @@ -35,4 +36,9 @@ public interface PipelineStep { * @param mappings Property mappings */ void execute(Map sourceData, Map outputData, Map mappings); + + /** Execute with the field catalog and other state belonging to this model only. */ + default void execute(ConversionContext context, Map mappings) { + execute(context.sourceData(), context.outputData(), mappings); + } } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java index 93528956..9464df4d 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java @@ -21,12 +21,21 @@ import static org.junit.jupiter.api.Assertions.*; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.ossie.app.OssieSalesforceConverter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class MetricCliTest { @TempDir @@ -57,4 +66,150 @@ void reportsMetricFailureToStderrWithoutWritingAModel() throws Exception { process.destroyForcibly(); } } + + @Test + void bindingsFlagWritesCompleteModelWithPhysicalOverrides() throws Exception { + Path input = inputFixture(); + String before = Files.readString(input); + Path bindings = write("bindings.yaml", """ + models: + Customer_Orders_Model: + dataspace: production + datasets: + Orders: + dataObjectName: OrdersProduction__dll + dataObjectType: Dlo + fields: + amount: NetRevenue__c + """); + CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); + assertEquals(0, result.exitCode(), result.stderr()); + JsonNode output = new ObjectMapper().readTree(Files.readString(directory.resolve("Customer_Orders_Model.json"))); + assertEquals("production", output.get("dataspace").asText()); + JsonNode orders = find(output.get("semanticDataObjects"), "Orders"); + assertEquals("OrdersProduction__dll", orders.get("dataObjectName").asText()); + assertEquals("NetRevenue__c", find(orders.get("semanticMeasurements"), "amount").get("dataObjectFieldName").asText()); + assertEquals("SUM([Orders].[amount])", find(output.get("semanticCalculatedMeasurements"), "total_revenue").get("expression").asText()); + assertEquals(before, Files.readString(input)); + } + + @ParameterizedTest + @ValueSource(strings = { + "models: {}\nmodels: {}", + "models: {Customer_Orders_Model: {expression: 'SUM(amount)'}}", + "models: {}\n---\nmodels: {Customer_Orders_Model: {dataspace: hidden}}" + }) + void invalidBindingsDocumentReportsInputErrorAndWritesNothing(String content) throws Exception { + Path input = inputFixture(); + Path bindings = write("bindings.yaml", content); + CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); + assertEquals(2, result.exitCode(), result.stderr()); + assertTrue(result.stderr().contains("Invalid Salesforce bindings"), result.stderr()); + assertFalse(result.stderr().contains("Exception in thread"), result.stderr()); + assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); + } + + @Test + void schemaInvalidBindingReportsConversionErrorWithoutStackTrace() throws Exception { + Path input = inputFixture(); + Path bindings = write("bindings.yaml", """ + models: + Customer_Orders_Model: + datasets: + Orders: {dataObjectType: NotANativeObjectType} + """); + CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); + assertEquals(3, result.exitCode(), result.stderr()); + assertTrue(result.stderr().contains("dataObjectType"), result.stderr()); + assertTrue(result.stderr().startsWith("Error:"), result.stderr()); + assertFalse(result.stderr().contains("Exception in thread"), result.stderr()); + assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); + } + + @Test + void rejectsBindingsFlagInReverseDirectionBeforeReadingBindings() throws Exception { + Path input = write("source.json", Files.readString(Path.of("src/test/resources/examples/salesforceToOssie.json"))); + CliResult result = run("toOssie", input.toString(), "--bindings", directory.resolve("nonexistent.yaml").toString()); + assertEquals(2, result.exitCode(), result.stderr()); + assertTrue(result.stderr().contains("Expected toSF"), result.stderr()); + try (var paths = Files.list(directory)) { + assertFalse(paths.anyMatch(path -> path.toString().endsWith(".yaml"))); + } + } + + @Test + void rejectsMissingBindingsPathAndUnknownFlags() throws Exception { + Path input = inputFixture(); + CliResult missingArgument = run("toSF", input.toString(), "--bindings"); + assertEquals(2, missingArgument.exitCode(), missingArgument.stderr()); + assertTrue(missingArgument.stderr().contains("Expected toSF"), missingArgument.stderr()); + CliResult missingFile = run("toSF", input.toString(), "--bindings", directory.resolve("missing.yaml").toString()); + assertEquals(2, missingFile.exitCode(), missingFile.stderr()); + assertTrue(missingFile.stderr().contains("Cannot read Salesforce bindings"), missingFile.stderr()); + CliResult unknown = run("toSF", input.toString(), "--mapping", "ignored"); + assertEquals(2, unknown.exitCode(), unknown.stderr()); + assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); + } + + @Test + void laterModelBindingFailurePreservesExistingOutputsAndWritesNoPartialModels() throws Exception { + ObjectMapper yaml = new ObjectMapper(new YAMLFactory()); + Map document = yaml.readValue(Files.readString( + Path.of("src/test/resources/examples/ossieToSalesforce.yaml")), new TypeReference<>() {}); + @SuppressWarnings("unchecked") Map first = (Map) ((List) document.get("semantic_model")).get(0); + Map second = new ObjectMapper().convertValue(first, new TypeReference<>() {}); + second.put("name", "Other_Model"); + document.put("semantic_model", List.of(first, second)); + Path input = write("input.yaml", yaml.writeValueAsString(document)); + String before = Files.readString(input); + Path existing = write("Customer_Orders_Model.json", "preserve existing model"); + Path bindings = write("bindings.yaml", """ + models: + Other_Model: + datasets: + Orders: + fields: {missing: InvalidColumn__c} + """); + CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); + assertEquals(3, result.exitCode(), result.stderr()); + assertTrue(result.stderr().contains("Other_Model"), result.stderr()); + assertTrue(result.stderr().contains("missing"), result.stderr()); + assertEquals("preserve existing model", Files.readString(existing)); + assertFalse(Files.exists(directory.resolve("Other_Model.json"))); + assertEquals(before, Files.readString(input)); + } + + private Path inputFixture() throws Exception { + return write("input.yaml", Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml"))); + } + + private Path write(String name, String content) throws Exception { + Path path = directory.resolve(name); + Files.writeString(path, content); + return path; + } + + private record CliResult(int exitCode, String stderr) {} + + private CliResult run(String... arguments) throws Exception { + List command = new ArrayList<>(List.of( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), OssieSalesforceConverter.class.getName())); + command.addAll(List.of(arguments)); + Path stderr = directory.resolve("stderr.txt"); + Process process = new ProcessBuilder(command).redirectError(stderr.toFile()) + .redirectOutput(directory.resolve("stdout.txt").toFile()).start(); + try { + assertTrue(process.waitFor(30, TimeUnit.SECONDS), "CLI did not terminate"); + return new CliResult(process.exitValue(), Files.readString(stderr)); + } finally { + process.destroyForcibly(); + } + } + + private static JsonNode find(JsonNode items, String name) { + for (JsonNode item : items) if (name.equals(item.path("apiName").asText())) return item; + throw new AssertionError("Missing exported item " + name); + } + } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java index 7a001777..c16bd761 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java @@ -25,6 +25,7 @@ import org.apache.ossie.converter.ConversionDirection; import org.apache.ossie.converter.Converter; import org.apache.ossie.converter.ConverterFactory; +import org.apache.ossie.converter.SalesforceBindings; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.exception.ValidationException; import org.apache.ossie.validator.SchemaValidator; @@ -71,6 +72,7 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() { + assumeTrue(salesforceSchemaExists, "Salesforce schema is required; see README setup instructions"); assumeTrue(ossieSchemaExists, "Ossie schema is required; see README setup instructions"); converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); } @@ -233,23 +235,18 @@ void fieldsFromAnotherSemanticModelCannotSatisfyAMetricReference() throws Except assertTrue(error.getMessage().contains("orders.profit"), error.getMessage()); } - @Test - void declaredButOmittedCalculatedSqlFieldCannotSatisfyAMetricReference() throws Exception { - Map source = model("sales", List.of()); + @ParameterizedTest + @ValueSource(strings = {"profit__c + 1", "profit__c+1"}) + void derivedSqlFieldIsExportedAndAvailableToMetricsRegardlessOfWhitespace(String expression) throws Exception { + Map source = model("sales", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); Map calculated = field("adjusted", "Decimal"); - calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", "profit__c + 1")))); + calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", expression)))); items(source, "datasets").get(0).put("fields", List.of(field("profit", "Decimal"), calculated)); Map output = convertOne(source); assertEquals(List.of("profit"), items(items(output, "semanticDataObjects").get(0), "semanticMeasurements").stream().map(item -> item.get("apiName")).toList()); - - source.put("metrics", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); - String input = document(List.of(source)); - ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); - - assertTrue(error.getMessage().contains("adjusted_total"), error.getMessage()); - assertTrue(error.getMessage().contains("orders.adjusted"), error.getMessage()); - assertTrue(error.getMessage().contains("not exported"), error.getMessage()); + assertEquals(1, items(output, "semanticCalculatedDimensions").size()); + assertEquals("SUM(([orders].[profit] + 1))", measurements(output).get(0).get("expression")); } @Test @@ -314,6 +311,109 @@ void translatedComposedMetricsValidateAgainstTheSalesforceSchema() throws Except assertThrows(ValidationException.class, () -> validator.validate(output)); } + @Test + void metricDependenciesResolveForwardReferencesAndRetainDeclarationOrder() throws Exception { + Map output = convertOne(model("sales", List.of( + metric("margin", "SNOWFLAKE", "total_profit / NULLIF(total_sales, 0)"), + metric("total_profit", "ANSI_SQL", "SUM(orders.profit)"), + metric("total_sales", "ANSI_SQL", "SUM(orders.revenue)")))); + assertEquals(List.of("margin", "total_profit", "total_sales"), measurements(output).stream().map(m -> m.get("apiName")).toList()); + String formula = measurements(output).get(0).get("expression").toString(); + assertTrue(formula.contains("SUM([orders].[profit])"), formula); + assertTrue(formula.contains("SUM([orders].[revenue])"), formula); + assertFalse(formula.contains("total_sales")); + } + + @Test + void nativeMetricMetadataSurvivesWhileFormulaMetadataIsCompiled() throws Exception { + Map metric = metric("money", "ANSI_SQL", "SUM(orders.profit)"); + metric.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + JSON.writeValueAsString(Map.of("label", "Profit in USD", "decimalPlace", 2, "dataType", "Currency", + "expression", "stale", "syntax", "Salesforce", "aggregationType", "Avg", "level", "Row"))))); + Map value = measurements(convertOne(model("sales", List.of(metric)))).get(0); + assertEquals("Profit in USD", value.get("label")); + assertEquals(2, value.get("decimalPlace")); + assertEquals("Currency", value.get("dataType")); + assertEquals("SUM([orders].[profit])", value.get("expression")); + assertEquals("Tua", value.get("syntax")); + assertEquals("UserAgg", value.get("aggregationType")); + assertEquals("AggregateFunction", value.get("level")); + } + + @Test + void incompatibleNativeMetricTypeFails() throws Exception { + Map metric = metric("money", "ANSI_SQL", "SUM(orders.profit)"); + metric.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataType\":\"Text\"}"))); + var error = assertThrows(ConversionException.class, () -> convertOne(model("sales", List.of(metric)))); + assertTrue(error.getMessage().contains("incompatible Salesforce dataType Text"), error.getMessage()); + } + + @Test + void outputSchemaIsEnforcedByPublicConversionAndWritesNothingOnFailure() throws Exception { + Map source = model("sales", List.of()); + source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataspace\":42}"))); + Path input = temporaryDirectory.resolve("bad-output.yaml"); + Files.writeString(input, document(List.of(source))); + Path output = Files.createDirectory(temporaryDirectory.resolve("output")); + assertThrows(ValidationException.class, () -> converter.convert(input, output)); + try (var files = Files.list(output)) { assertEquals(0, files.count()); } + } + + @Test + void sameConverterCanRecoverAfterFailureWithoutLeakingDependencyState() throws Exception { + Map bad = model("sales", List.of(metric("a", "ANSI_SQL", "b + 1"), metric("b", "ANSI_SQL", "a + 1"))); + assertThrows(ConversionException.class, () -> convertOne(bad)); + Map good = model("sales", List.of(metric("a", "ANSI_SQL", "b + 1"), metric("b", "ANSI_SQL", "SUM(orders.profit)"))); + assertEquals(convertOne(good), convertOne(good)); + } + + @Test + void bindingsRetargetPhysicalObjectsWithoutChangingTheOsiDocumentOrDerivedReferences() throws Exception { + Map source = model("sales", List.of(metric("total", "ANSI_SQL", "SUM(orders.adjusted)"))); + Map calculated = field("adjusted", "Decimal"); + calculated.put("expression", Map.of("dialects", List.of(dialect("SNOWFLAKE", "profit__c + 1")))); + items(source, "datasets").get(0).put("fields", List.of(field("profit", "Decimal"), calculated)); + String document = document(List.of(source)); + SalesforceBindings bindings = SalesforceBindings.fromString(""" + models: + sales: + dataspace: analytics + datasets: + orders: + dataObjectName: OrdersProduction__dll + dataObjectType: Dlo + fields: + profit: NetProfit__c + """); + Converter bound = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings); + Map output = parse(bound.convert(document).get(0)); + assertEquals("analytics", output.get("dataspace")); + Map dataset = items(output, "semanticDataObjects").get(0); + assertEquals("OrdersProduction__dll", dataset.get("dataObjectName")); + assertEquals("NetProfit__c", items(dataset, "semanticMeasurements").get(0).get("dataObjectFieldName")); + assertEquals("SUM(([orders].[profit] + 1))", measurements(output).get(0).get("expression")); + assertEquals(document, document(List.of(source))); + assertEquals(output, parse(bound.convert(document).get(0))); + } + + @Test + void unusedDirectFieldCannotSilentlyChangeDatatype() throws Exception { + Map source = model("sales", List.of()); + items(items(source, "datasets").get(0), "fields").get(0).put("custom_extensions", + List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataType\":\"Text\"}"))); + var error = assertThrows(ConversionException.class, () -> convertOne(source)); + assertTrue(error.getMessage().contains("conflicts"), error.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"Time", "Opaque"}) + void unsupportedDirectDatatypeNeedsExplicitNativeMapping(String datatype) throws Exception { + Map source = model("sales", List.of()); + items(items(source, "datasets").get(0), "fields").get(0).put("datatype", datatype); + var error = assertThrows(ConversionException.class, () -> convertOne(source)); + assertTrue(error.getMessage().contains("no safe Salesforce mapping"), error.getMessage()); + } + private Map convertOne(Map model) throws IOException { List outputs = converter.convert(document(List.of(model))); assertEquals(1, outputs.size()); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 99cd754f..f11622af 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -86,6 +86,7 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() throws IOException { + assumeTrue(salesforceSchemaExists, "Salesforce schema is required; see README setup instructions."); assumeTrue(ossieSchemaExists, "Ossie schema file is required but not found. See README for setup instructions."); converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); @@ -224,11 +225,15 @@ void testCalculatedFieldDetection() throws Exception { Map ansiModel = jsonMapper.readValue(ansiResults.get(0), new TypeReference>() {}); List> ansiCalcDimensions = (List>) ansiModel.get("semanticCalculatedDimensions"); - assertNull(ansiCalcDimensions, "ANSI_SQL dialect: no semanticCalculatedDimensions"); + assertNotNull(ansiCalcDimensions); + assertEquals(2, ansiCalcDimensions.size()); + assertTrue(ansiCalcDimensions.stream().allMatch(field -> "Tua".equals(field.get("syntax")))); + assertTrue(ansiCalcDimensions.stream().anyMatch(field -> field.get("expression").toString().contains("MID("))); + assertTrue(ansiCalcDimensions.stream().anyMatch(field -> field.get("expression").toString().contains("YEAR("))); } @Test - void testInvalidRelationshipsFiltered() throws Exception { + void testAllDeclaredRelationshipsAreExported() throws Exception { List results = converter.convert(ossieYaml); Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); @@ -245,6 +250,21 @@ void testInvalidRelationshipsFiltered() throws Exception { assertTrue(hasValidOrdersProducts, "Orders_Products should be included"); } + @Test + void testCalculatedRelationshipKeyFailsInsteadOfDroppingTheRelationship() { + String invalid = ossieYaml.replace(" metrics:", """ + - name: Orders_ByYear + from: Orders + to: Products + from_columns: [order_year] + to_columns: [product_id] + metrics:""".indent(4).stripTrailing()); + Exception error = assertThrows(org.apache.ossie.exception.ConversionException.class, + () -> converter.convert(invalid)); + assertTrue(error.getMessage().contains("Orders_ByYear"), error.getMessage()); + assertTrue(error.getMessage().contains("order_year"), error.getMessage()); + } + @Test void testCustomExtensionsRestoration() throws Exception { List results = converter.convert(ossieYaml); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java index ffc77647..665944fc 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java @@ -178,8 +178,8 @@ void testFieldMapping() throws Exception { List> dialects = (List>) expression.get("dialects"); assertNotNull(dialects); assertEquals(1, dialects.size()); - assertEquals("TABLEAU", dialects.get(0).get("dialect")); - assertEquals("customer_id__c", dialects.get(0).get("expression")); + assertEquals("ANSI_SQL", dialects.get(0).get("dialect")); + assertEquals("\"customer_id__c\"", dialects.get(0).get("expression")); } @Test diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java new file mode 100644 index 00000000..ee817f47 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ConstantFieldMetricTest { + private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper JSON = new ObjectMapper(); + + @ParameterizedTest + @ValueSource(strings = {"ANSI_SQL", "SNOWFLAKE", "TABLEAU"}) + void rejectsSummingConstantRowFieldsWhoseDatasetWouldDisappear(String dialect) throws Exception { + Map document = fixture(); + Map model = model(document); + for (String dataset : List.of("Orders", "Products")) { + addField(model, dataset, field("one_per_row", dialect, "1")); + } + // Each individual metric needs a different row population; SUM(1) cannot express this. + for (String dataset : List.of("Orders", "Products")) { + String reference = dialect.equals("TABLEAU") ? "[" + dataset + "].[one_per_row]" : dataset + ".one_per_row"; + model.put("metrics", List.of(metric("row_population", dialect, "SUM(" + reference + ")"))); + ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); + assertTrue(error.getMessage().contains("Metric 'row_population'"), error.getMessage()); + assertTrue(error.getMessage().contains(dataset + ".one_per_row"), error.getMessage()); + assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"COUNT", "COUNTD", "MIN", "MAX", "AVG"}) + void doesNotAllowOtherAggregatesToBypassConstantFieldScopeChecks(String function) throws Exception { + Map document = fixture(); + Map model = model(document); + addField(model, "Orders", field("one_per_row", "TABLEAU", "1 + 0")); + model.put("metrics", List.of(metric("count_rows", "TABLEAU", function + "([Orders].[one_per_row])"))); + ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); + assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"ANSI_SQL", "SNOWFLAKE", "TABLEAU"}) + void stillExportsStandaloneConstantFieldsAndConstantMetrics(String dialect) throws Exception { + Map document = fixture(); + Map model = model(document); + addField(model, "Orders", field("constant", dialect, "1 + 1")); + addField(model, "Products", field("constant", dialect, "2")); + model.put("metrics", List.of(metric("fixed_value", dialect, "42"))); + Map output = convert(document); + assertEquals("42", items(output, "semanticCalculatedMeasurements").get(0).get("expression")); + assertEquals(2, items(output, "semanticCalculatedDimensions").stream() + .filter(item -> List.of("Orders__constant", "Products__constant").contains(item.get("apiName"))).count()); + } + + @Test + void derivedRowExpressionCanCombineAConstantFieldWithAPhysicalField() throws Exception { + Map document = fixture(); + Map model = model(document); + addField(model, "Orders", field("constant_one", "ANSI_SQL", "1")); + Map amountPlusOne = field("amount_plus_one", "ANSI_SQL", "amount + constant_one"); + amountPlusOne.put("datatype", "Decimal"); + addField(model, "Orders", amountPlusOne); + model.put("metrics", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(Orders.amount_plus_one)"))); + String expression = items(convert(document), "semanticCalculatedMeasurements").get(0).get("expression").toString(); + assertTrue(expression.contains("[Orders].[amount]"), expression); + assertTrue(expression.contains("+ 1"), expression); + } + + @Test + void constantDependenciesCannotSmuggleDatasetScopeThroughAnAlias() throws Exception { + Map document = fixture(); + Map model = model(document); + addField(model, "Orders", field("constant_one", "ANSI_SQL", "1")); + addField(model, "Orders", field("constant_alias", "TABLEAU", "[Orders].[constant_one] + 0")); + model.put("metrics", List.of(metric("total", "ANSI_SQL", "SUM(Orders.constant_alias)"))); + ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); + assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); + } + + private static Map field(String name, String dialect, String expression) { + return new LinkedHashMap<>(Map.of("name", name, "datatype", "Integer", "expression", + Map.of("dialects", List.of(Map.of("dialect", dialect, "expression", expression))))); + } + + private static Map metric(String name, String dialect, String expression) { + Map metric = field(name, dialect, expression); + metric.put("datatype", "Decimal"); + return metric; + } + + private static void addField(Map model, String datasetName, Map field) { + Map dataset = items(model, "datasets").stream() + .filter(item -> datasetName.equals(item.get("name"))).findFirst().orElseThrow(); + List> fields = new ArrayList<>(items(dataset, "fields")); + fields.add(field); + dataset.put("fields", fields); + } + + private static Map fixture() throws Exception { + return YAML.readValue(Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")), new TypeReference<>() {}); + } + + private static Map model(Map document) { + return items(document, "semantic_model").get(0); + } + + private static Map convert(Map document) throws Exception { + String output = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE) + .convert(YAML.writeValueAsString(document)).get(0); + return JSON.readValue(output, new TypeReference<>() {}); + } + + @SuppressWarnings("unchecked") + private static List> items(Map map, String key) { + return (List>) map.getOrDefault(key, List.of()); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java new file mode 100644 index 00000000..1145ce57 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; + +class CustomExtensionHandlerTest { + private final ObjectMapper json = new ObjectMapper(); + private final CustomExtensionHandler handler = new CustomExtensionHandler(json); + + @ParameterizedTest + @ValueSource(strings = {"{", "{} {}", "{} null", "{\"label\":\"a\",\"label\":\"b\"}", + "{\"nested\":{\"x\":1,\"x\":2}}", "null", "[]", "42", "\"text\""}) + void rejectsMalformedAmbiguousOrNonObjectJsonWithoutWritingPartialProperties(String data) { + Map target = new LinkedHashMap<>(Map.of("apiName", "owner")); + ConversionException error = assertThrows(ConversionException.class, + () -> handler.restoreSalesforceCustomExtension(target, item("owner", data))); + assertTrue(error.getMessage().contains("owner"), error.getMessage()); + assertTrue(error.getMessage().contains("JSON object"), error.getMessage()); + assertEquals(Map.of("apiName", "owner"), target); + } + + @ParameterizedTest + @NullSource + @ValueSource(ints = {1}) + void rejectsMissingOrNonStringData(Object data) { + ConversionException error = assertThrows(ConversionException.class, + () -> handler.restoreSalesforceCustomExtension(new LinkedHashMap<>(), item("owner", data))); + assertTrue(error.getMessage().contains("owner")); + assertTrue(error.getMessage().contains("encoded as a string")); + } + + @Test + void preservesNativePropertiesAndCorePrecedenceWithoutChangingSharedMapper() { + Map target = new LinkedHashMap<>(Map.of("label", "core")); + handler.restoreSalesforceCustomExtension(target, + item("owner", "{\"label\":\"native\",\"description\":\"kept\",\"nested\":{\"x\":1}}")); + assertEquals("core", target.get("label")); + assertEquals("kept", target.get("description")); + assertEquals(Map.of("x", 1), target.get("nested")); + assertFalse(json.isEnabled(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)); + assertFalse(json.isEnabled(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)); + } + + @Test + void ignoresOtherVendorData() { + Map source = new LinkedHashMap<>(Map.of("name", "owner", "custom_extensions", + List.of(Map.of("vendor_name", "SNOWFLAKE", "data", "not Salesforce JSON")))); + Map target = new LinkedHashMap<>(); + assertDoesNotThrow(() -> handler.restoreSalesforceCustomExtension(target, source)); + assertTrue(target.isEmpty()); + } + + @ParameterizedTest + @ValueSource(strings = {"model", "dataset", "field", "metric"}) + void publicConversionRejectsMalformedMetadataAtEveryOwnerLevel(String ownerKind) throws Exception { + Map field = new LinkedHashMap<>(Map.of("name", "amount", "datatype", "Decimal", + "expression", dialect("amount__c"))); + Map dataset = item("orders", "{\"dataObjectType\":\"Dlo\"}"); + dataset.put("source", "orders__dll"); + dataset.put("fields", List.of(field)); + Map metric = new LinkedHashMap<>(Map.of("name", "total", "datatype", "Decimal", + "expression", dialect("SUM(orders.amount)"))); + Map model = item("sales", "{\"dataspace\":\"default\"}"); + model.put("datasets", List.of(dataset)); + model.put("metrics", List.of(metric)); + Map owner = switch (ownerKind) { + case "model" -> model; + case "dataset" -> dataset; + case "field" -> field; + default -> metric; + }; + owner.put("custom_extensions", item("ignored", "{").get("custom_extensions")); + String input = json.writeValueAsString(Map.of("version", "0.2.0.dev0", "semantic_model", List.of(model))); + ConversionException error = assertThrows(ConversionException.class, + () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); + assertTrue(error.getMessage().contains((String) owner.get("name")), error.getMessage()); + assertTrue(error.getMessage().contains("JSON"), error.getMessage()); + } + + private static Map dialect(String expression) { + return Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", expression))); + } + + private static Map item(String name, Object data) { + Map extension = new LinkedHashMap<>(); + extension.put("vendor_name", "SALESFORCE"); + extension.put("data", data); + return new LinkedHashMap<>(Map.of("name", name, "custom_extensions", List.of(extension))); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java new file mode 100644 index 00000000..ab4ebacb --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ExpressionCompilerTest { + private static final Map TYPES = Map.of( + "amount", "Decimal", "quantity", "Integer", "email", "String", "active", "Boolean", + "ordered", "Date", "timestamp", "DateTime", "zoned", "DateTimeTz"); + private static ExpressionCompiler.Binding resolve(ExpressionCompiler.Reference reference) { + String field = reference.parts().getLast().text(); + String datatype = TYPES.get(field); + if (datatype == null) throw new IllegalArgumentException("Unknown field " + field); + return new ExpressionCompiler.Binding("[orders].[" + field + "]", datatype, "orders"); + } + private static ExpressionCompiler.Compiled compile(String source, String dialect) { + return ExpressionCompiler.compile(ExpressionCompiler.parse(source, dialect), ExpressionCompilerTest::resolve); + } + + @Test void parsesWithoutBindingAndRetainsQuotedIdentifiers() { + var parsed = ExpressionCompiler.parse("\"Order.Items\".\"Net Revenue\"", "SNOWFLAKE"); + var reference = ExpressionCompiler.directReference(parsed).orElseThrow(); + assertEquals(List.of(new MetricFieldResolver.Identifier("Order.Items", true), + new MetricFieldResolver.Identifier("Net Revenue", true)), reference.parts()); + assertFalse(reference.tableau()); + assertTrue(ExpressionCompiler.directReference(ExpressionCompiler.parse("(amount)", "ANSI_SQL")).isPresent()); + assertTrue(ExpressionCompiler.directReference(ExpressionCompiler.parse("amount + 1", "ANSI_SQL")).isEmpty()); + } + + @Test void independentlyBindsAndRendersTheSameImmutableParse() { + var parsed = ExpressionCompiler.parse("SUM(amount)", "SNOWFLAKE"); + AtomicInteger calls = new AtomicInteger(); + var first = ExpressionCompiler.compile(parsed, reference -> { + calls.incrementAndGet(); return new ExpressionCompiler.Binding("[one].[net]", "Decimal", "one"); + }); + var second = ExpressionCompiler.compile(parsed, + reference -> new ExpressionCompiler.Binding("[two].[gross]", "Decimal", "two")); + assertEquals(1, calls.get()); + assertEquals("SUM([one].[net])", first.expression()); + assertEquals(Set.of("two"), second.datasets()); + assertEquals("SUM([two].[gross])", second.expression()); + } + + @ParameterizedTest + @ValueSource(strings = {"CAST(amount AS INT)", "amount::INT", "SUM(amount) OVER ()", + "SUM(amount) FILTER (WHERE active)", "(SELECT amount FROM orders)", + "amount IN (1, 2)", "SUM(amount ORDER BY quantity)", "amount(+) = quantity", + "CASE amount WHEN 1 THEN 2 END", "SUM(amount) AS alias", "SUM(amount); SUM(quantity)", + "SUM(amount) trailing", "SELECT amount", "unknown_function(amount)", "COALESCE()"}) + void rejectsShapesWithoutExplicitCapabilities(String source) { + assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); + } + + @ParameterizedTest + @ValueSource(strings = {"SUM(ALL amount)", "SUM(UNIQUE amount)", "SUM(amount IGNORE NULLS)", + "SUM(amount RESPECT NULLS)", "SUM(amount) IGNORE NULLS", + "SUM(amount) KEEP (DENSE_RANK LAST ORDER BY quantity)", "SUM(amount LIMIT 1)", + "SUM(amount HAVING MAX quantity)", "SUM(amount ORDER BY quantity)", + "SUM(amount).attribute", "private_schema.SUM(amount)", "\"SUM\"(amount)", + "N'prefixed'", "email ISNULL", "email NOTNULL", "PRIOR amount = quantity", "!active"}) + void rejectsDialectModifiersInsteadOfSilentlyDiscardingThem(String source) { + assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); + } + + @Test void nativeMetricReferencesAreBoundByTheModelResolver() { + var result = ExpressionCompiler.compile(ExpressionCompiler.parse("[net] + 1", "TABLEAU"), reference -> { + assertTrue(reference.tableau()); + assertEquals(List.of(new MetricFieldResolver.Identifier("net", true)), reference.parts()); + return new ExpressionCompiler.Binding("SUM([orders].[amount])", "Decimal", "orders", ExpressionCompiler.Level.AGGREGATE); + }); + assertEquals("(SUM([orders].[amount]) + 1)", result.expression()); + assertEquals(ExpressionCompiler.Level.AGGREGATE, result.level()); + } + + @Test void escapedQuotedIdentifiersDoNotChangeReferenceBoundaries() { + for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { + var reference = ExpressionCompiler.directReference(ExpressionCompiler.parse("\"a\"\"b.c\".\"d\"\"e\"", dialect)).orElseThrow(); + assertEquals(List.of(new MetricFieldResolver.Identifier("a\"b.c", true), + new MetricFieldResolver.Identifier("d\"e", true)), reference.parts()); + } + var bracket = ExpressionCompiler.directReference(ExpressionCompiler.parse("[a.b].[c]", "ANSI_SQL")).orElseThrow(); + assertTrue(bracket.tableau()); + assertEquals(List.of(new MetricFieldResolver.Identifier("a.b", true), + new MetricFieldResolver.Identifier("c", true)), bracket.parts()); + // JSqlParser does not consume doubled closing brackets. Fail closed, rather + // than resolving a truncated identifier to another field. + assertThrows(IllegalArgumentException.class, + () -> ExpressionCompiler.parse("[a.b].[c]]d]", "ANSI_SQL")); + } + + @Test void preservesNumberPrecisionAndEscapedStringValues() { + assertEquals("0.12345678901234567890123456789", + compile("0.12345678901234567890123456789", "SNOWFLAKE").expression()); + var result = compile("CASE WHEN email = 'O''Brien -- not a comment' THEN 'a''b' ELSE 'x' END", "SNOWFLAKE"); + assertEquals("String", result.datatype()); + assertEquals(ExpressionCompiler.Level.ROW, result.level()); + assertEquals("(IF ([orders].[email] = 'O''Brien -- not a comment') THEN 'a''b' ELSE 'x' END)", result.expression()); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void compilesExistingDerivedDateAndEmailDomainExpressions(String dialect) { + var year = compile("YEAR(ordered)", dialect); + assertEquals("YEAR([orders].[ordered])", year.expression()); + assertEquals("Integer", year.datatype()); + var domain = compile("SUBSTRING(email, POSITION('@' IN email) + 1, LENGTH(email))", dialect); + assertEquals("MID([orders].[email], (FIND([orders].[email], '@') + 1), LEN([orders].[email]))", domain.expression()); + assertEquals("String", domain.datatype()); + assertEquals(ExpressionCompiler.Level.ROW, domain.level()); + assertEquals(domain.expression(), compile(domain.expression(), "TABLEAU").expression()); + assertEquals("SUM(YEAR([orders].[ordered]))", compile("SUM(YEAR(ordered))", dialect).expression()); + assertEquals("SUM(LEN([orders].[email]))", compile("SUM(LENGTH(email))", dialect).expression()); + } + + @ParameterizedTest + @ValueSource(strings = {"SUBSTRING(email, 0)", "SUBSTRING(email, -1)", "SUBSTRING(email, quantity)", + "SUBSTRING(email, 1, -1)", "SUBSTRING(email, 1, quantity)", "SUBSTRING(email, 1.5)", + "YEAR(zoned)", "YEAR(email)", "LENGTH(quantity)", "POSITION(1 IN email)"}) + void rejectsScalarDomainsWithoutEquivalentTargetSemantics(String source) { + assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); + } + + @ParameterizedTest + @ValueSource(strings = {"CEIL(AVG(amount))", "FLOOR(AVG(amount))", "ROUND(AVG(amount))", + "ROUND(AVG(amount), 0)", "ROUND(AVG(amount), -2)"}) + void integralRoundingCanSatisfyAnIntegerMetricDeclaration(String source) { + var result = compile(source, "SNOWFLAKE"); + assertEquals("Integer", result.datatype()); + var metric = Map.of("name", "rounded", "datatype", "Integer", "expression", + Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", source)))); + assertEquals("Integer", MetricExpressionTranslator.compile(metric, ExpressionCompilerTest::resolve).datatype()); + } + + @Test void positivePrecisionDoesNotClaimAnIntegralResult() { + assertEquals("Decimal", compile("ROUND(AVG(amount), 2)", "SNOWFLAKE").datatype()); + assertEquals("Integer", compile("ROUND(SUM(quantity), 2)", "SNOWFLAKE").datatype()); + } + + @Test void boundDerivedRowsAndMetricsPreserveTheirLevelAndLineage() { + var row = ExpressionCompiler.compile(ExpressionCompiler.parse("SUM(net)", "SNOWFLAKE"), + reference -> new ExpressionCompiler.Binding("([orders].[amount] * [orders].[quantity])", "Decimal", "orders")); + assertEquals("SUM(([orders].[amount] * [orders].[quantity]))", row.expression()); + var aggregate = ExpressionCompiler.compile(ExpressionCompiler.parse("ratio + 1", "SNOWFLAKE"), + reference -> new ExpressionCompiler.Binding("(SUM([orders].[amount]) / SUM([costs].[amount]))", + "Decimal", Set.of("orders", "costs"), ExpressionCompiler.Level.AGGREGATE)); + assertEquals(Set.of("orders", "costs"), aggregate.datasets()); + assertEquals(ExpressionCompiler.Level.AGGREGATE, aggregate.level()); + assertThrows(IllegalArgumentException.class, () -> ExpressionCompiler.compile( + ExpressionCompiler.parse("SUM(ratio)", "SNOWFLAKE"), + reference -> new ExpressionCompiler.Binding("SUM([orders].[amount])", "Decimal", "orders", ExpressionCompiler.Level.AGGREGATE))); + } + + @Test void declaredTypeSurvivesAnAllNullMetricForDependentMetrics() { + var metric = Map.of("name", "nullable", "datatype", "Decimal", "expression", + Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "NULL")))); + assertEquals("Decimal", MetricExpressionTranslator.compile(metric, ExpressionCompilerTest::resolve).datatype()); + } + + @Test void limitsInputDepthTokensAndEmittedExpansion() { + assertThrows(IllegalArgumentException.class, () -> compile("(".repeat(129) + "1" + ")".repeat(129), "SNOWFLAKE")); + assertThrows(IllegalArgumentException.class, () -> compile("1+".repeat(5000) + "1", "SNOWFLAKE")); + assertThrows(IllegalArgumentException.class, () -> compile("1".repeat(32769), "SNOWFLAKE")); + String growing = "SUM(amount)"; + for (int i = 0; i < 18; i++) growing = "NULLIF(" + growing + ", 0)"; + String source = growing; + var exception = assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE")); + assertTrue(exception.getMessage().contains("131072"), exception.getMessage()); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java new file mode 100644 index 00000000..50006ab3 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class FieldExpressionPlanTest { + @ParameterizedTest + @ValueSource(strings = {"amount__c+profit__c", "amount__c + profit__c"}) + void parsesDerivedFieldsIndependentOfWhitespaceAndBindsPhysicalColumns(String expression) { + ConversionContext context = export(dataset("Orders", "warehouse.sales.orders", + field("amount", "Decimal", "amount__c"), field("profit", "Decimal", "profit__c"), + field("derived", "Decimal", expression))); + assertEquals(2, direct(context, "Orders").size()); + Map calc = calculated(context).get(0); + assertEquals("Orders__derived", calc.get("apiName")); + assertEquals("Tua", calc.get("syntax")); + assertEquals("Number", calc.get("dataType")); + assertTrue(calc.get("expression").toString().contains("[Orders].[amount]")); + assertTrue(calc.get("expression").toString().contains("[Orders].[profit]")); + assertFalse(calc.get("expression").toString().contains("__c")); + assertEquals(2, ((List) calc.get("dependencies")).size()); + assertFalse(context.fieldPlan().isDirect("Orders", "derived")); + } + + @ParameterizedTest + @ValueSource(strings = {"cost+tax", "unit(price)", "gross/net", "Order Date", "has\"quote", "SUM", "schema.column"}) + void preservesQuotedPhysicalNamesInsteadOfClassifyingTheirCharacters(String physical) { + ConversionContext context = export(dataset("Orders", "orders", + field("value", "Decimal", "\"" + physical.replace("\"", "\"\"") + "\""))); + assertEquals(physical, direct(context, "Orders").get(0).get("dataObjectFieldName")); + assertTrue(calculated(context).isEmpty()); + } + + @Test + void extractsOnlyPhysicalColumnFromVerifiedSourceQualifier() { + ConversionContext context = export(dataset("Orders", "warehouse.sales.orders", + field("amount", "Decimal", "sales.orders.amount__c"))); + assertEquals("amount__c", direct(context, "Orders").get(0).get("dataObjectFieldName")); + assertFailure("qualifier", dataset("Orders", "warehouse.sales.orders", + field("amount", "Decimal", "another_table.amount__c"))); + } + + @Test + void expandsForwardReferencesAndCachesResolvedRowBindings() { + ConversionContext context = export(dataset("Orders", "orders", + field("gross", "Decimal", "net + tax"), + field("net", "Decimal", "amount - discount"), + field("amount", "Decimal", "amount__c"), + field("discount", "Decimal", "discount__c"), + field("tax", "Decimal", "tax__c"))); + ExpressionCompiler.Binding binding = context.fieldPlan().resolve("Orders", "gross"); + assertSame(binding, context.fieldPlan().resolve("Orders", "gross")); + assertTrue(binding.expression().contains("[Orders].[amount]")); + assertTrue(binding.expression().contains("[Orders].[discount]")); + assertTrue(binding.expression().contains("[Orders].[tax]")); + assertFalse(binding.expression().contains("[net]")); + assertEquals(ExpressionCompiler.Level.ROW, binding.level()); + assertEquals(3, context.fieldPlan().dependencies("Orders", "gross").size()); + } + + @Test + void rejectsCyclesWithWholeDependencyPath() { + assertFailure("Orders.a -> Orders.b -> Orders.a", dataset("Orders", "orders", + field("a", "Decimal", "b + 1"), field("b", "Decimal", "a + 1"))); + assertFailure("dependency cycle", dataset("Orders", "orders", field("a", "Decimal", "a + 1"))); + } + + @Test + void rejectsUnknownOrAmbiguousPhysicalAndSemanticReferences() { + assertFailure("Unknown row field reference", dataset("Orders", "orders", + field("derived", "Decimal", "undeclared + 1"))); + assertFailure("Ambiguous row field reference", dataset("Orders", "orders", + field("one", "Decimal", "amount__c"), field("two", "Decimal", "amount__c"), + field("derived", "Decimal", "amount__c + 1"))); + assertFailure("Ambiguous row field reference", dataset("Orders", "orders", + field("one", "Decimal", "amount"), field("amount", "Decimal", "second__c"), + field("derived", "Decimal", "amount + 1"))); + } + + @Test + void respectsQuotedPhysicalIdentifierCase() { + ConversionContext context = export(dataset("Orders", "orders", + field("amount", "Decimal", "\"mixedCase\""), + field("derived", "Decimal", "\"mixedCase\" + 1"))); + assertTrue(context.fieldPlan().resolve("Orders", "derived").expression().contains("[Orders].[amount]")); + assertFailure("Unknown row field reference", dataset("Orders", "orders", + field("amount", "Decimal", "\"mixedCase\""), + field("derived", "Decimal", "mixedCase + 1"))); + } + + @Test + void rejectsUnsupportedUnusedExpressionsRatherThanOmittingThem() { + assertFailure("Field 'Orders.derived'", dataset("Orders", "orders", + field("amount", "Decimal", "amount__c"), + field("derived", "Decimal", "UNKNOWN_FUNCTION(amount)"))); + } + + @Test + void rejectsAggregateFieldsAndIncompatibleDeclaredResultTypes() { + assertFailure("row expressions", dataset("Orders", "orders", + field("amount", "Decimal", "amount__c"), field("derived", "Decimal", "SUM(amount)"))); + assertFailure("conflicts", dataset("Orders", "orders", + field("amount", "Decimal", "amount__c"), field("derived", "String", "amount + 1"))); + } + + @Test + void infersStringBooleanAndNumericRowResultTypes() { + ConversionContext context = export(dataset("Orders", "orders", + field("amount", "Decimal", "amount__c"), field("name", "String", "name__c"), + field("positive", null, "amount > 0"), + field("category", null, "CASE WHEN amount > 0 THEN 'positive' ELSE name END"), field("constant", null, "42"))); + assertEquals("Boolean", context.fieldPlan().resolve("Orders", "positive").datatype()); + assertEquals("String", context.fieldPlan().resolve("Orders", "category").datatype()); + assertEquals("Integer", context.fieldPlan().resolve("Orders", "constant").datatype()); + } + + @Test + void treatsTableauDirectReferenceAsSemanticAliasNotPhysicalColumn() { + Map alias = field("alias", "Decimal", "[Orders].[amount]"); + alias.put("expression", expression("TABLEAU", "[Orders].[amount]")); + ConversionContext context = export(dataset("Orders", "orders", + field("amount", "Decimal", "amount__c"), alias)); + assertEquals(1, direct(context, "Orders").size()); + assertEquals("[Orders].[amount]", context.fieldPlan().resolve("Orders", "alias").expression()); + assertEquals("Orders__alias", calculated(context).get(0).get("apiName")); + } + + @Test + void rejectsCrossDatasetRowFieldsEvenWhenTheOtherDatasetIsDeclared() { + Map cross = field("cross", "Decimal", "[Returns].[amount] + 1"); + cross.put("expression", expression("TABLEAU", "[Returns].[amount] + 1")); + assertFailure("cross-dataset", dataset("Orders", "orders", cross), + dataset("Returns", "returns", field("amount", "Decimal", "amount__c"))); + assertFailure("qualifier", dataset("Orders", "orders", field("cross", "Decimal", "Returns.amount + 1")), + dataset("Returns", "returns", field("amount", "Decimal", "amount__c"))); + } + + @Test + void usesStableDistinctGlobalNamesAcrossDatasetsAndSanitizationCollisions() { + Map a = dataset("a-b", "a", field("value", "Integer", "1 + 1")); + Map b = dataset("a_b", "b", field("value", "Integer", "2 + 2")); + ConversionContext first = export(a, b); + ConversionContext reversed = export(b, a); + String firstName = first.fieldPlan().calculatedApiName("a-b", "value"); + String secondName = first.fieldPlan().calculatedApiName("a_b", "value"); + assertNotEquals(firstName, secondName); + assertTrue(firstName.matches("[A-Za-z_][A-Za-z0-9_]*")); + assertEquals(firstName, reversed.fieldPlan().calculatedApiName("a-b", "value")); + assertEquals(secondName, reversed.fieldPlan().calculatedApiName("a_b", "value")); + } + + @Test + void reservesMetricNamesBeforeAllocatingCalculatedDimensionNames() { + Map dataset = dataset("Orders", "orders", field("constant", "Integer", "1 + 1")); + Map source = new LinkedHashMap<>(Map.of("datasets", List.of(dataset), + "metrics", List.of(Map.of("name", "Orders__constant")))); + ConversionContext context = exportModel(source); + assertNotEquals("Orders__constant", context.fieldPlan().calculatedApiName("Orders", "constant")); + } + + @Test + void rejectsDuplicateDeclarationsAndMissingTargetDataset() { + assertFailure("Duplicate field", dataset("Orders", "orders", + field("amount", "Decimal", "one__c"), field("amount", "Decimal", "two__c"))); + Map source = Map.of("datasets", List.of(dataset("Orders", "orders", field("a", "Integer", "a")))); + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> handler().execute(source, new LinkedHashMap<>(), Map.of())); + assertTrue(error.getMessage().contains("was not exported"), error.getMessage()); + } + + @Test + void reverseConversionQuotesPhysicalColumnsAndRoundTripsTheirExactName() { + String physical = "gross/+ \"net\""; + Map sfDataset = new LinkedHashMap<>(Map.of("apiName", "Orders", + "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number", "dataObjectFieldName", physical)))); + Map source = new LinkedHashMap<>(Map.of("semanticDataObjects", List.of(sfDataset))); + Map dataset = new LinkedHashMap<>(Map.of("name", "Orders", "source", "orders")); + Map output = new LinkedHashMap<>(Map.of("datasets", List.of(dataset))); + new FieldMappingHandler(ConversionDirection.SALESFORCE_TO_OSSIE, + new CustomExtensionHandler(new ObjectMapper())).execute(source, output, Map.of()); + ConversionContext context = export(dataset); + assertEquals(physical, direct(context, "Orders").get(0).get("dataObjectFieldName")); + } + + @Test + void preservesQuotedQualifiedPhysicalColumnsAndRejectsWrongQualifierCase() { + String source = "\"Warehouse\".\"SaLes\".\"Order Items\""; + ConversionContext context = export(dataset("Orders", source, + field("amount", "Decimal", source + ".\"Net Value\""), + field("double_amount", "Decimal", "\"SaLes\".\"Order Items\".\"Net Value\" * 2"))); + assertEquals("Net Value", direct(context, "Orders").get(0).get("dataObjectFieldName")); + assertTrue(context.fieldPlan().resolve("Orders", "double_amount").expression().contains("[Orders].[amount]")); + assertFailure("qualifier", dataset("Orders", source, + field("amount", "Decimal", "\"WAREHOUSE\".\"SaLes\".\"Order Items\".\"Net Value\""))); + } + + @Test + void rejectsExcessiveExpansionAndDependencyDepthWithUsefulErrors() { + List> exponential = new ArrayList<>(); + exponential.add(field("f0", "Integer", "1 + 1")); + for (int i = 1; i < 20; i++) { + exponential.add(field("f" + i, "Integer", "f" + (i - 1) + " + f" + (i - 1))); + } + assertFailure("translated expression exceeds 131072 characters", new LinkedHashMap<>(Map.of( + "name", "Orders", "source", "orders", "fields", exponential))); + + List> deep = new ArrayList<>(); + for (int i = 0; i < 130; i++) deep.add(field("f" + i, "Integer", "f" + (i + 1) + " + 1")); + deep.add(field("f130", "Integer", "1 + 1")); + assertFailure("dependency depth exceeds 128", new LinkedHashMap<>(Map.of( + "name", "Orders", "source", "orders", "fields", deep))); + } + + @Test + void physicalBindingOverlayLeavesSourceAndCompiledDependenciesUnchanged() throws Exception { + Map source = new LinkedHashMap<>(Map.of("name", "Retail", "datasets", List.of( + dataset("Orders", "warehouse.orders", field("amount", "Decimal", "amount__c"), + field("derived", "Decimal", "amount__c + 1"))))); + ObjectMapper mapper = new ObjectMapper(); + String original = mapper.writeValueAsString(source); + ConversionContext context = exportModel(source); + ExpressionCompiler.Binding binding = context.fieldPlan().resolve("Orders", "derived"); + List> dependencies = context.fieldPlan().dependencies("Orders", "derived"); + SalesforceBindings.fromString(""" + models: + Retail: + dataspace: prod + datasets: + Orders: + dataObjectName: Orders__dlm + dataObjectType: DataModelObject + fields: + amount: NetRevenue__c + """).apply(source, context.outputData()); + assertEquals(original, mapper.writeValueAsString(source)); + assertEquals("NetRevenue__c", direct(context, "Orders").get(0).get("dataObjectFieldName")); + assertSame(binding, context.fieldPlan().resolve("Orders", "derived")); + assertEquals(dependencies, context.fieldPlan().dependencies("Orders", "derived")); + assertTrue(binding.expression().contains("[Orders].[amount]")); + assertFalse(binding.expression().contains("NetRevenue__c")); + assertEquals(List.of(Map.of("dependentDefinitionApiName", "Orders", "dependentFieldApiName", "amount")), dependencies); + } + + @Test + void calculatedRowMetadataOverridesStaleNativeFormulaProperties() throws Exception { + Map derived = field("derived", "Decimal", "amount + 1"); + derived.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + new ObjectMapper().writeValueAsString(Map.of("level", "AggregateFunction", "syntax", "Salesforce", + "expression", "SUM(stale)", "apiName", "renamed", "label", "Adjusted amount", "dataType", "Currency", + "dependencies", List.of(Map.of("dependentDefinitionApiName", "Wrong", "dependentFieldApiName", "missing"))))))); + ConversionContext context = export(dataset("Orders", "orders", field("amount", "Decimal", "amount__c"), derived)); + Map calc = calculated(context).get(0); + assertEquals("Row", calc.get("level")); + assertEquals("Tua", calc.get("syntax")); + assertEquals("Orders__derived", calc.get("apiName")); + assertEquals("Adjusted amount", calc.get("label")); + assertEquals("Currency", calc.get("dataType")); + assertTrue(calc.get("expression").toString().contains("[Orders].[amount]")); + assertFalse(calc.get("expression").toString().contains("SUM")); + assertEquals(List.of(Map.of("dependentDefinitionApiName", "Orders", "dependentFieldApiName", "amount")), calc.get("dependencies")); + } + + @SafeVarargs + private static void assertFailure(String message, Map... datasets) { + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () -> export(datasets)); + assertTrue(error.getMessage().contains(message), error.getMessage()); + } + + @SafeVarargs + private static ConversionContext export(Map... datasets) { + return exportModel(new LinkedHashMap<>(Map.of("datasets", List.of(datasets)))); + } + + private static ConversionContext exportModel(Map source) { + List targets = new ArrayList<>(); + for (Object item : (List) source.get("datasets")) { + Map dataset = (Map) item; + targets.add(new LinkedHashMap<>(Map.of("apiName", dataset.get("name")))); + } + Map target = new LinkedHashMap<>(Map.of("semanticDataObjects", targets)); + ConversionContext context = new ConversionContext(source, target); + handler().execute(context, Map.of()); + return context; + } + + private static FieldMappingHandler handler() { + return new FieldMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, + new CustomExtensionHandler(new ObjectMapper())); + } + + @SuppressWarnings("unchecked") + private static List> calculated(ConversionContext context) { + return (List>) context.outputData().getOrDefault("semanticCalculatedDimensions", List.of()); + } + + @SuppressWarnings("unchecked") + private static List> direct(ConversionContext context, String dataset) { + return ((List>) context.outputData().get("semanticDataObjects")).stream() + .filter(item -> dataset.equals(item.get("apiName"))).flatMap(item -> List.of("semanticDimensions", "semanticMeasurements") + .stream().flatMap(key -> ((List>) item.getOrDefault(key, List.of())).stream())).toList(); + } + + @SafeVarargs + private static Map dataset(String name, String source, Map... fields) { + return new LinkedHashMap<>(Map.of("name", name, "source", source, "fields", List.of(fields))); + } + + private static Map field(String name, String datatype, String text) { + Map field = new LinkedHashMap<>(); + field.put("name", name); + if (datatype != null) field.put("datatype", datatype); + field.put("expression", expression("ANSI_SQL", text)); + return field; + } + + private static Map expression(String dialect, String text) { + return Map.of("dialects", List.of(Map.of("dialect", dialect, "expression", text))); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java new file mode 100644 index 00000000..e3e6a965 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; + +class MetricCompilationPlanTest { + private static Map metric(String name, String expression) { + return metric(name, expression, "ANSI_SQL"); + } + private static Map metric(String name, String expression, String dialect) { + return new LinkedHashMap<>(Map.of("name", name, "expression", Map.of("dialects", + List.of(Map.of("dialect", dialect, "expression", expression))))); + } + private static MetricCompilationPlan plan(List> metrics) { + Map source = Map.of("metrics", metrics, "datasets", List.of(Map.of("name", "orders", "fields", + List.of(Map.of("name", "profit", "datatype", "Decimal"), Map.of("name", "units", "datatype", "Integer"))))); + Map target = Map.of("semanticDataObjects", List.of(Map.of("apiName", "orders", "semanticMeasurements", + List.of(Map.of("apiName", "profit", "dataType", "Number"), Map.of("apiName", "units", "dataType", "Number"))))); + return new MetricCompilationPlan(source, new MetricFieldResolver(source, target)); + } + @Test + void dependenciesKeepAggregateLevelAndInferredDatatypeAndAreCached() { + var plan = plan(List.of(metric("ratio", "total / count_units"), metric("total", "SUM(orders.profit)"), + metric("count_units", "COUNT(orders.units)"))); + var result = plan.compile("ratio"); + assertEquals(ExpressionCompiler.Level.AGGREGATE, result.level()); + assertEquals("Decimal", result.datatype()); + assertEquals(java.util.Set.of("orders"), result.datasets()); + assertEquals("Integer", plan.compile("count_units").datatype()); + assertSame(result, plan.compile("ratio")); + } + @Test + void bracketedTableauMetricReferencesUseTheSameDependencyChecks() { + var plan = plan(List.of(metric("result", "[total] + 1", "TABLEAU"), metric("total", "SUM(orders.profit)"))); + assertEquals("((SUM([orders].[profit])) + 1)", plan.compile("result").expression()); + } + @Test + void forwardReferencesAndNormalizedSqlNamesAreResolved() { + var plan = plan(List.of(metric("result", "TOTAL + 1"), metric("total", "SUM(orders.profit)"))); + assertTrue(plan.compile("result").expression().contains("SUM([orders].[profit])")); + } + @Test + void ambiguousUnqualifiedFieldOrMetricIsRejected() { + var plan = plan(List.of(metric("result", "profit + 1"), metric("profit", "SUM(orders.profit)"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("result")).getMessage().contains("ambiguous field or metric")); + assertDoesNotThrow(() -> plan.compile("profit")); + } + @Test + void ambiguousCaseFoldedMetricNamesAreRejected() { + var plan = plan(List.of(metric("result", "total + 1"), metric("total", "1"), metric("TOTAL", "2"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("result")).getMessage().contains("ambiguous")); + } + @Test + void reportsDependencyCycleWithPathAndCanStillCompileIndependentMetric() { + var plan = plan(List.of(metric("a", "b + 1"), metric("b", "c + 1"), metric("c", "a + 1"), metric("ok", "1"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("a")).getMessage().contains("a -> b -> c -> a")); + assertEquals("1", plan.compile("ok").expression()); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("a")).getMessage().contains("a -> b -> c -> a")); + } + @Test + void rejectsAggregateOfAnAggregateMetric() { + var plan = plan(List.of(metric("bad", "SUM(total)"), metric("total", "SUM(orders.profit)"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("bad")).getMessage().contains("nested aggregate")); + } + @Test + void rejectsRowAndAggregateMixAcrossDependency() { + var plan = plan(List.of(metric("bad", "total + orders.profit"), metric("total", "SUM(orders.profit)"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("bad")).getMessage().contains("mix aggregate")); + } + @Test + void allNullDependencyRetainsItsExplicitDeclaredType() { + var empty = metric("empty", "NULL"); + empty.put("datatype", "Integer"); + var plan = plan(List.of(metric("result", "COALESCE(empty, 1)"), empty)); + assertEquals("Integer", plan.compile("result").datatype()); + } + @Test + void fractionalDependencyCannotSatisfyIntegerDeclaration() { + var integer = metric("rounded", "total"); + integer.put("datatype", "Integer"); + var plan = plan(List.of(integer, metric("total", "AVG(orders.units)"))); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("rounded")).getMessage().contains("incompatible")); + } + @Test + void boundsDependencyDepthBeforeStackExhaustion() { + List> metrics = new ArrayList<>(); + for (int i = 0; i < 150; i++) metrics.add(metric("m" + i, i == 149 ? "1" : "m" + (i + 1) + " + 1")); + var plan = plan(metrics); + assertTrue(assertThrows(ConversionException.class, () -> plan.compile("m0")).getMessage().contains("dependency depth")); + } + @Test + void boundsExponentialDependencyExpansion() { + List> metrics = new ArrayList<>(); + metrics.add(metric("m0", "SUM(orders.profit)")); + for (int i = 1; i < 20; i++) metrics.add(metric("m" + i, "m" + (i - 1) + " + m" + (i - 1))); + var plan = plan(metrics); + var error = assertThrows(ConversionException.class, () -> plan.compile("m19")); + assertTrue(error.getMessage().contains("exceeds 131072 characters"), error.getMessage()); + } + @Test + void sharedDependencyAcrossManyMetricsDoesNotAlterTheCompiledResult() { + List> metrics = new ArrayList<>(); + metrics.add(metric("base", "SUM(orders.profit)")); + for (int i = 0; i < 500; i++) metrics.add(metric("m" + i, "base + " + i)); + var plan = plan(metrics); + for (int i = 0; i < 500; i++) assertEquals("((SUM([orders].[profit])) + " + i + ")", plan.compile("m" + i).expression()); + assertSame(plan.compile("base"), plan.compile("base")); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java index 7e045fca..ee8407c3 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java @@ -22,7 +22,12 @@ import static org.junit.jupiter.api.Assertions.*; import java.math.BigDecimal; +import java.math.MathContext; import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -169,6 +174,43 @@ void textPredicatesPreserveDuplicatesNullsAndEscapedApostrophes(String dialect) assertValue(dialect, "COUNT(DISTINCT orders.status)", List.of(row(null, null, null, null)), 0.0); } + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void dateAndEmailDomainRowExpressionsEvaluateAtBoundaryValues(String dialect) { + Map types = Map.of("date", "Date", "timestamp", "DateTime", "email", "String", "amount", "Decimal"); + Map row = new LinkedHashMap<>(); + row.put("date", LocalDate.of(2024, 2, 29)); + row.put("timestamp", LocalDateTime.of(2025, 1, 1, 0, 0)); + row.put("amount", new BigDecimal("0.1000000000000000000000001")); + assertScalar(dialect, "YEAR(date)", row, types, new BigDecimal("2024")); + assertScalar(dialect, "YEAR(timestamp)", row, types, new BigDecimal("2025")); + row.put("date", null); + assertScalar(dialect, "YEAR(date)", row, types, null); + String domain = "SUBSTRING(email, POSITION('@' IN email) + 1, LENGTH(email))"; + for (String address : List.of("alice@example.com", "no-at-sign", "@", "", "a@b@c")) { + row.put("email", address); + assertScalar(dialect, domain, row, types, address.substring(address.indexOf('@') + 1)); + } + row.put("email", null); + assertScalar(dialect, domain, row, types, null); + assertScalar(dialect, "amount = 0.1000000000000000000000001", row, types, Boolean.TRUE); + assertScalar(dialect, "amount + 0.2", row, types, new BigDecimal("0.3000000000000000000000001")); + assertScalar(dialect, "NULLIF(amount, 0.1000000000000000000000001)", row, types, null); + } + + private static void assertScalar(String dialect, String source, Map row, + Map types, Object expected) { + var compiled = ExpressionCompiler.compile(ExpressionCompiler.parse(source, dialect), reference -> { + String name = reference.parts().getLast().text(); + return new ExpressionCompiler.Binding("[orders].[" + name + "]", types.get(name), "orders"); + }); + Object actual = new TuaSubsetEvaluator(compiled.expression()).evaluateRow(row); + if (expected instanceof BigDecimal number) { + assertInstanceOf(Number.class, actual); + assertEquals(0, number.compareTo(new BigDecimal(actual.toString())), source); + } else assertEquals(expected, actual, source); + } + private static void assertValue( String dialect, String sql, List> rows, Double expected) { Map metric = Map.of( @@ -244,6 +286,12 @@ Object evaluate(List> rows) { return calculation.value(rows, Map.of()); } + Object evaluateRow(Map row) { + Calculation calculation = expression(0); + assertEquals(tokens.size(), position, "Unconsumed generated Tua tokens"); + return calculation.value(List.of(row), row); + } + private Calculation expression(int minimum) { Calculation left = prefix(); while (position < tokens.size() && precedence(tokens.get(position)) >= minimum) { @@ -275,7 +323,7 @@ private Calculation prefix() { Calculation child = expression(token.equals("-") ? 7 : 3); return (rows, row) -> { Object value = child.value(rows, row); - return value == null ? null : token.equals("-") ? -number(value) : !(Boolean) value; + return value == null ? null : token.equals("-") ? decimal(value).negate() : !(Boolean) value; }; } if (token.equalsIgnoreCase("NULL")) { @@ -300,7 +348,7 @@ private Calculation prefix() { }; } if (Character.isDigit(token.charAt(0))) { - return (rows, row) -> Double.valueOf(token); + return (rows, row) -> new BigDecimal(token); } expect("("); List arguments = new ArrayList<>(); @@ -319,24 +367,26 @@ private static Calculation function(String name, List arguments) { .map(input -> arguments.getFirst().value(rows, input)) .filter(java.util.Objects::nonNull).toList(); if (name.equals("COUNT")) { - return (double) values.size(); + return BigDecimal.valueOf(values.size()); } if (name.equals("COUNTD")) { - return (double) values.stream().distinct().count(); + return BigDecimal.valueOf(values.stream().map(value -> value instanceof Number + ? decimal(value).stripTrailingZeros() : value).distinct().count()); } if (values.isEmpty()) { return null; } return switch (name) { - case "SUM" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).sum(); - case "AVG" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).average().orElseThrow(); - case "MIN" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).min().orElseThrow(); - case "MAX" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).max().orElseThrow(); + case "SUM" -> values.stream().map(TuaSubsetEvaluator::decimal).reduce(BigDecimal.ZERO, BigDecimal::add); + case "AVG" -> values.stream().map(TuaSubsetEvaluator::decimal).reduce(BigDecimal.ZERO, BigDecimal::add) + .divide(BigDecimal.valueOf(values.size()), MathContext.DECIMAL128); + case "MIN" -> values.stream().min(TuaSubsetEvaluator::compare).orElseThrow(); + case "MAX" -> values.stream().max(TuaSubsetEvaluator::compare).orElseThrow(); default -> throw new AssertionError(name); }; }; } - assertTrue(List.of("IFNULL", "ISNULL", "ABS", "CEILING", "FLOOR", "ROUND").contains(name), + assertTrue(List.of("IFNULL", "ISNULL", "ABS", "CEILING", "FLOOR", "ROUND", "YEAR", "LEN", "FIND", "MID").contains(name), "Unsupported generated function: " + name); return (rows, row) -> { Object value = arguments.getFirst().value(rows, row); @@ -351,13 +401,28 @@ private static Calculation function(String name, List arguments) { if (value == null) { return null; } + if (name.equals("FIND")) { + Object needle = arguments.get(1).value(rows, row); + return needle == null ? null : BigDecimal.valueOf(((String) value).indexOf((String) needle) + 1); + } + if (name.equals("MID")) { + Object startValue = arguments.get(1).value(rows, row); + Object lengthValue = arguments.size() > 2 ? arguments.get(2).value(rows, row) : null; + if (startValue == null || arguments.size() > 2 && lengthValue == null) return null; + int start = decimal(startValue).intValueExact() - 1; + String string = (String) value; + if (start >= string.length()) return ""; + int end = arguments.size() > 2 ? Math.min(string.length(), start + decimal(lengthValue).intValueExact()) : string.length(); + return string.substring(start, end); + } return switch (name) { - case "ABS" -> Math.abs(number(value)); - case "CEILING" -> Math.ceil(number(value)); - case "FLOOR" -> Math.floor(number(value)); - case "ROUND" -> BigDecimal.valueOf(number(value)).setScale( - arguments.size() == 1 ? 0 : (int) number(arguments.get(1).value(rows, row)), - RoundingMode.HALF_UP).doubleValue(); + case "YEAR" -> BigDecimal.valueOf(((TemporalAccessor) value).get(ChronoField.YEAR)); + case "LEN" -> BigDecimal.valueOf(((String) value).length()); + case "ABS" -> decimal(value).abs(); + case "CEILING" -> decimal(value).setScale(0, RoundingMode.CEILING); + case "FLOOR" -> decimal(value).setScale(0, RoundingMode.FLOOR); + case "ROUND" -> decimal(value).setScale( + arguments.size() == 1 ? 0 : decimal(arguments.get(1).value(rows, row)).intValueExact(), RoundingMode.HALF_UP); default -> throw new AssertionError("Unsupported generated function: " + name); }; }; @@ -380,25 +445,31 @@ private static Object binary(String operator, Object left, Object right) { return null; } return switch (operator) { - case "+" -> number(left) + number(right); - case "-" -> number(left) - number(right); - case "*" -> number(left) * number(right); + case "+" -> decimal(left).add(decimal(right)); + case "-" -> decimal(left).subtract(decimal(right)); + case "*" -> decimal(left).multiply(decimal(right)); case "/" -> { - assertNotEquals(0.0, number(right), "Generated expression evaluated an unguarded zero divisor"); - yield number(left) / number(right); + assertNotEquals(0, decimal(right).signum(), "Generated expression evaluated an unguarded zero divisor"); + yield decimal(left).divide(decimal(right), MathContext.DECIMAL128); } - case "=" -> left.equals(right); - case "!=", "<>" -> !left.equals(right); - case "<" -> number(left) < number(right); - case "<=" -> number(left) <= number(right); - case ">" -> number(left) > number(right); - case ">=" -> number(left) >= number(right); + case "=" -> compare(left, right) == 0; + case "!=", "<>" -> compare(left, right) != 0; + case "<" -> compare(left, right) < 0; + case "<=" -> compare(left, right) <= 0; + case ">" -> compare(left, right) > 0; + case ">=" -> compare(left, right) >= 0; default -> throw new AssertionError(operator); }; } - private static double number(Object value) { - return ((Number) value).doubleValue(); + private static BigDecimal decimal(Object value) { + return value instanceof BigDecimal decimal ? decimal : new BigDecimal(value.toString()); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static int compare(Object left, Object right) { + if (left instanceof Number && right instanceof Number) return decimal(left).compareTo(decimal(right)); + return ((Comparable) left).compareTo(right); } private static int precedence(String token) { diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java index 79d9d832..9bfc151a 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java @@ -159,13 +159,11 @@ static Stream unsupported() { Arguments.of("COALESCE(SUM(orders.amount))", "expects 2"), Arguments.of("NULLIF(SUM(orders.amount))", "expects 2"), Arguments.of("MEDIAN(orders.amount)", "unsupported function"), - Arguments.of("CAST(orders.amount AS DECIMAL)", "expected )"), - Arguments.of("SUM(YEAR(orders.ordered))", "unsupported function"), - Arguments.of("SUM(LENGTH(orders.status))", "unsupported function"), - Arguments.of("SUM(orders.amount) OVER ()", "unexpected token"), - Arguments.of("SUM(orders.amount) FILTER (WHERE orders.active)", "unexpected token"), + Arguments.of("CAST(orders.amount AS DECIMAL)", "unsupported SQL expression CastExpression"), + Arguments.of("SUM(orders.amount) OVER ()", "unsupported SQL expression AnalyticExpression"), + Arguments.of("SUM(orders.amount) FILTER (WHERE orders.active)", "unsupported SQL expression AnalyticExpression"), Arguments.of("{ FIXED : SUM(orders.amount) }", "unsupported character"), - Arguments.of("SELECT SUM(orders.amount)", "Unknown field"), + Arguments.of("SELECT SUM(orders.amount)", "unexpected token"), Arguments.of("SUM(orders.amount);", "unsupported character"), Arguments.of("SUM(orders.amount) -- comment", "comments"), Arguments.of("SUM(orders.amount) /* comment */", "comments"), diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java new file mode 100644 index 00000000..eaf8ab3e --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class RelationshipMappingHandlerTest { + private final CustomExtensionHandler extensions = new CustomExtensionHandler(new ObjectMapper()); + private final RelationshipMappingHandler handler = + new RelationshipMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, extensions); + + @Test + void invalidFirstRelationshipFailsWithoutMisassigningItsKeysToTheSecond() { + Map bad = relation("bad", "computed"); + Map good = relation("good", "customer_id"); + Map source = source(bad, good); + Map target = target(); + Map mappings = mappings(); + ConversionException error = assertThrows(ConversionException.class, + () -> handler.execute(source, target, mappings)); + assertTrue(error.getMessage().contains("bad")); + assertTrue(error.getMessage().contains("calculated join keys")); + assertEquals(List.of(bad, good), source.get("relationships"), "Do not filter or mutate source relationships"); + assertFalse(target.containsKey("semanticRelationships"), "No partially mapped valid relationship is published"); + assertFalse(mappings.containsKey("relationships")); + } + + @Test + void allInvalidRelationshipsCannotFallThroughToGenericRawMapping() { + Map source = source(relation("bad", "computed")); + Map target = target(); + Map mappings = mappings(); + assertThrows(ConversionException.class, () -> handler.execute(source, target, mappings)); + new SemanticModelMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, extensions) + .execute(source, target, mappings); + assertFalse(target.containsKey("semanticRelationships")); + } + + @Test + void mapsEveryCompositeKeyPairAndUsesOssieDirectionForDefaultCardinality() { + Map relation = relation("join", "customer_id"); + relation.put("from_columns", List.of("customer_id", "region")); + relation.put("to_columns", List.of("id", "region")); + Map target = target(); + handler.execute(source(relation), target, mappings()); + Map exported = relationships(target).get(0); + assertEquals("ManyToOne", exported.get("cardinality")); + assertEquals(true, exported.get("isEnabled")); + assertEquals("Auto", exported.get("joinType")); + assertEquals(List.of(Map.of("leftSemanticFieldApiName", "customer_id", "rightSemanticFieldApiName", "id"), + Map.of("leftSemanticFieldApiName", "region", "rightSemanticFieldApiName", "region")), exported.get("criteria")); + } + + @ParameterizedTest + @ValueSource(strings = {"OneToOne", "OneToMany", "ManyToOne", "ManyToMany", "Unspecified"}) + void preservesExplicitNativeCardinalityForSalesforceRoundTrips(String cardinality) { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"cardinality\":\"" + cardinality + "\",\"isEnabled\":false,\"joinType\":\"Left\"}"))); + Map target = target(); + handler.execute(source(relation), target, mappings()); + Map exported = relationships(target).get(0); + assertEquals(cardinality, exported.get("cardinality")); + assertEquals(false, exported.get("isEnabled")); + assertEquals("Left", exported.get("joinType")); + } + + @Test + void rejectsInvalidNativeCardinalityInsteadOfSilentlyReplacingIt() { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"cardinality\":\"Guess\"}"))); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("cardinality")); + } + + @ParameterizedTest + @ValueSource(strings = {"cardinality", "isEnabled", "joinType"}) + void doesNotReplaceExplicitNullNativeMetadataWithDefaults(String property) { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"" + property + "\":null}"))); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source(relation), target(), mappings())).getMessage().contains(property)); + } + + @Test + void rejectsCompositeArityMismatch() { + Map relation = relation("join", "customer_id"); + relation.put("from_columns", List.of("customer_id", "region")); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("same number")); + } + + @ParameterizedTest + @ValueSource(strings = {"from", "to"}) + void rejectsUnknownEndpoint(String endpoint) { + Map relation = relation("join", "customer_id"); + relation.put(endpoint, "missing"); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("missing")); + } + + @Test + void rejectsDuplicateRelationNamesAndRepeatedKeys() { + assertTrue(assertThrows(ConversionException.class, () -> handler.execute( + source(relation("join", "customer_id"), relation("JOIN", "customer_id")), target(), mappings())) + .getMessage().contains("Duplicate")); + Map relation = relation("join", "customer_id"); + relation.put("from_columns", List.of("customer_id", "customer_id")); + relation.put("to_columns", List.of("id", "region")); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("duplicate field")); + } + + @Test + void declaredUniquenessMustCoverTheRelationshipTargetKey() { + Map source = source(relation("join", "customer_id")); + datasets(source).get(1).put("primary_key", List.of("region")); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source, target(), mappings())).getMessage().contains("declared primary_key")); + datasets(source).get(1).put("unique_keys", List.of(List.of("id"))); + Map target = target(); + assertDoesNotThrow(() -> handler.execute(source, target, mappings())); + assertFalse(relationships(target).get(0).containsKey("primaryNameField")); + } + + @Test + void nativeOneToManyChecksLeftUniquenessAndAllowsNonUniqueRightKey() { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"cardinality\":\"OneToMany\"}"))); + Map source = source(relation); + datasets(source).get(0).put("primary_key", List.of("customer_id")); + datasets(source).get(1).put("primary_key", List.of("region")); + assertDoesNotThrow(() -> handler.execute(source, target(), mappings())); + datasets(source).get(0).put("primary_key", List.of("region")); + assertTrue(assertThrows(ConversionException.class, + () -> handler.execute(source, target(), mappings())).getMessage().contains("from_columns")); + } + + @ParameterizedTest + @ValueSource(strings = {"ManyToMany", "Unspecified"}) + void nativeNonUniqueCardinalitiesDoNotInventKeyConstraints(String cardinality) { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"cardinality\":\"" + cardinality + "\"}"))); + Map source = source(relation); + datasets(source).get(0).put("primary_key", List.of("region")); + datasets(source).get(1).put("primary_key", List.of("region")); + assertDoesNotThrow(() -> handler.execute(source, target(), mappings())); + } + + @Test + void coreJoinKeysRemainAuthoritativeOverStaleNativeExtensions() { + Map relation = relation("join", "customer_id"); + relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"criteria\":[{\"leftSemanticFieldApiName\":\"stale\",\"rightSemanticFieldApiName\":\"stale\"}]}"))); + Map target = target(); + handler.execute(source(relation), target, mappings()); + assertEquals(List.of(Map.of("leftSemanticFieldApiName", "customer_id", "rightSemanticFieldApiName", "id")), + relationships(target).get(0).get("criteria")); + } + + @Test + void salesforceToOssieKeepsNativeCardinalityAndPairOrder() { + Map sf = target(); + handler.execute(source(relation("join", "customer_id")), sf, mappings()); + relationships(sf).get(0).put("cardinality", "OneToMany"); + Map osi = new LinkedHashMap<>(); + Map reverseMappings = new LinkedHashMap<>(); + reverseMappings.put("semanticRelationships", "relationships"); + reverseMappings.put("semanticRelationships.apiName", "relationships.name"); + new RelationshipMappingHandler(ConversionDirection.SALESFORCE_TO_OSSIE, extensions) + .execute(sf, osi, reverseMappings); + @SuppressWarnings("unchecked") + Map relation = ((List>) osi.get("relationships")).get(0); + assertEquals("orders", relation.get("from")); + assertEquals(List.of("customer_id"), relation.get("from_columns")); + assertEquals(List.of("id"), relation.get("to_columns")); + assertTrue(relation.get("custom_extensions").toString().contains("OneToMany")); + } + + private static Map source(Map... relationships) { + Map source = new LinkedHashMap<>(); + source.put("name", "sales"); + source.put("datasets", new ArrayList<>(List.of( + sourceDataset("orders", "customer_id", "region", "computed"), sourceDataset("customers", "id", "region")))); + source.put("relationships", new ArrayList<>(List.of(relationships))); + return source; + } + + private static Map sourceDataset(String name, String... fields) { + Map result = new LinkedHashMap<>(); + result.put("name", name); + result.put("fields", java.util.Arrays.stream(fields).map(field -> Map.of("name", field)).toList()); + return result; + } + + private static Map target() { + Map target = new LinkedHashMap<>(); + target.put("semanticDataObjects", List.of( + Map.of("apiName", "orders", "semanticDimensions", List.of(Map.of("apiName", "customer_id"), Map.of("apiName", "region"))), + Map.of("apiName", "customers", "semanticDimensions", List.of(Map.of("apiName", "id"), Map.of("apiName", "region"))))); + return target; + } + + private static Map relation(String name, String field) { + return new LinkedHashMap<>(Map.of("name", name, "from", "orders", "to", "customers", + "from_columns", List.of(field), "to_columns", List.of("id"))); + } + + private static Map mappings() { + Map mappings = new LinkedHashMap<>(); + mappings.put("name", "apiName"); + mappings.put("relationships", "semanticRelationships"); + mappings.put("relationships.name", "semanticRelationships.apiName"); + return mappings; + } + + @SuppressWarnings("unchecked") + private static List> relationships(Map target) { + return (List>) target.get("semanticRelationships"); + } + + @SuppressWarnings("unchecked") + private static List> datasets(Map source) { + return (List>) source.get("datasets"); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java new file mode 100644 index 00000000..593406b3 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.apache.ossie.exception.InvalidInputException; +import org.apache.ossie.exception.ValidationException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class SalesforceBindingsTest { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); + private static final String MODEL = "Customer_Orders_Model"; + @TempDir Path directory; + + @Test + void acceptsEmptyCatalogAndLoadsJsonOrYamlFromAFile() throws Exception { + assertTrue(SalesforceBindings.none().isEmpty()); + assertTrue(SalesforceBindings.fromString("models: {}").isEmpty()); + Path file = directory.resolve("bindings.json"); + Files.writeString(file, "{\"models\":{\"sales\":{\"dataspace\":\"production\"}}}"); + SalesforceBindings bindings = SalesforceBindings.fromPath(file); + Map target = new LinkedHashMap<>(); + bindings.apply(Map.of("name", "sales"), target); + assertEquals("production", target.get("dataspace")); + assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromPath(directory.resolve("missing.yaml"))); + } + + @ParameterizedTest + @ValueSource(strings = { + "models: {}\nmodels: {}", + "models: {sales: {}, sales: {}}", + "models: {sales: {dataspace: a, dataspace: b}}", + "models: {sales: {datasets: {orders: {}, orders: {}}}}", + "models: {sales: {datasets: {orders: {fields: {amount: a, amount: b}}}}}" + }) + void rejectsDuplicateKeysAtEveryBindingsLevel(String input) { + InvalidInputException error = assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); + assertTrue(error.getMessage().contains("Duplicate field"), error.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = { + "models: {}\nunknown: true", + "models: {sales: {metrics: {}}}", + "models: {sales: {expression: 'SUM(amount)'}}", + "models: {sales: {datasets: {orders: {apiName: changed}}}}", + "models: {sales: {datasets: {orders: {expression: 'amount + 1'}}}}", + "models: {sales: {datasets: {orders: {syntax: Tua}}}}" + }) + void rejectsUnknownPropertiesIncludingFormulaAndSemanticNameOverrides(String input) { + InvalidInputException error = assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); + assertTrue(error.getMessage().contains("unknown property"), error.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = { + "models: {}\n---\nmodels: {sales: {dataspace: hidden}}", + "models: {}\n---\nnull", + "{\"models\":{}}\n{\"models\":{}}" + }) + void rejectsTrailingYamlDocumentsAndJsonValues(String input) { + assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); + } + + @ParameterizedTest + @ValueSource(strings = { + "", "null", "[]", "{}", "models: null", "models: []", "models: {sales: null}", + "models: {'': {}}", "models: {sales: {datasets: null}}", + "models: {sales: {dataspace: 5}}", "models: {sales: {dataspace: ''}}", + "models: {sales: {datasets: {orders: {fields: {amount: {expression: 'amount + 1'}}}}}}", + "models: {sales: {datasets: {orders: {fields: {amount: null}}}}}", + "models: {sales: {datasets: {orders: {fields: {amount: 17}}}}}", + "models: {sales: {datasets: {orders: {dataObjectName: ' '}}}}", + "{\"models\":{\"sales\":{\"dataspace\":\"bad\\nname\"}}}" + }) + void rejectsInvalidShapesAndNonStringOrBlankBindingValues(String input) { + assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); + } + + @Test + void absentModelBindingLeavesTargetUntouchedAndNamesMatchExactly() { + SalesforceBindings bindings = SalesforceBindings.fromString("models: {sales: {dataspace: production}}"); + Map target = new LinkedHashMap<>(Map.of("dataspace", "original")); + bindings.apply(Map.of("name", "Sales"), target); + assertEquals(Map.of("dataspace", "original"), target); + assertThrows(ConversionException.class, () -> bindings.validateModels(java.util.Set.of("Sales"))); + } + + @Test + void appliesPhysicalBindingsWithoutMutatingCanonicalModelOrFormulaMetadata() throws Exception { + String fixture = fixture(); + Map original = YAML.readValue(fixture, new TypeReference<>() {}); + Map baseline = output(SalesforceBindings.none(), fixture); + SalesforceBindings bindings = SalesforceBindings.fromString(""" + models: + Customer_Orders_Model: + dataspace: production + datasets: + Orders: + dataObjectName: OrdersProduction__dll + dataObjectType: Dlo + fields: + amount: NetRevenue__c + order_id: OrderIdentifier__c + """); + Map bound = output(bindings, fixture); + assertEquals("production", bound.get("dataspace")); + Map orders = find(items(bound, "semanticDataObjects"), "Orders"); + assertEquals("OrdersProduction__dll", orders.get("dataObjectName")); + assertEquals("NetRevenue__c", find(items(orders, "semanticMeasurements"), "amount").get("dataObjectFieldName")); + assertEquals("OrderIdentifier__c", find(items(orders, "semanticDimensions"), "order_id").get("dataObjectFieldName")); + assertEquals(baseline.get("semanticCalculatedMeasurements"), bound.get("semanticCalculatedMeasurements")); + assertEquals(baseline.get("semanticCalculatedDimensions"), bound.get("semanticCalculatedDimensions")); + assertEquals(baseline.get("semanticRelationships"), bound.get("semanticRelationships")); + assertEquals(original, YAML.readValue(fixture, new TypeReference>() {})); + // Exercise apply directly against the original map, in addition to the string API. + @SuppressWarnings("unchecked") Map model = (Map) ((List) original.get("semantic_model")).get(0); + String before = JSON.writeValueAsString(original); + bindings.apply(model, baseline); + assertEquals(before, JSON.writeValueAsString(original)); + assertEquals(bound, baseline); + assertEquals(bound, output(bindings, fixture), "bindings must be reusable without conversion state"); + } + + @Test + void rejectsUnknownModelDatasetAndFieldIdentitiesThroughPublicConverter() throws Exception { + String fixture = fixture(); + assertConversionError("unknown model 'missing'", "models: {missing: {dataspace: prod}}", fixture); + assertConversionError("dataset 'Missing'", "models: {" + MODEL + ": {datasets: {Missing: {dataObjectName: x}}}}", fixture); + assertConversionError("field 'missing'", "models: {" + MODEL + ": {datasets: {Orders: {fields: {missing: x}}}}}", fixture); + assertConversionError("dataset 'orders'", "models: {" + MODEL + ": {datasets: {orders: {dataObjectName: x}}}}", fixture); + } + + @Test + void rejectsCalculatedFieldPhysicalMappingsThroughPublicConverter() throws Exception { + assertConversionError("only direct physical fields can be rebound", + "models: {" + MODEL + ": {datasets: {Orders: {fields: {order_year: CalendarYear__c}}}}}", fixture()); + } + + @Test + void nativeSchemaRejectsUnsupportedDataObjectTypeAfterApplyingBindings() throws Exception { + SalesforceBindings bindings = SalesforceBindings.fromString( + "models: {" + MODEL + ": {datasets: {Orders: {dataObjectType: NotANativeObjectType}}}}"); + String fixture = fixture(); + ValidationException error = assertThrows(ValidationException.class, () -> output(bindings, fixture)); + assertTrue(error.getMessage().contains("dataObjectType"), error.getMessage()); + } + + @Test + void oneBindingCatalogCanTargetMultipleModelsWithoutCrossModelLeakage() throws Exception { + Map document = YAML.readValue(fixture(), new TypeReference<>() {}); + @SuppressWarnings("unchecked") Map first = (Map) ((List) document.get("semantic_model")).get(0); + Map second = JSON.convertValue(first, new TypeReference<>() {}); + second.put("name", "Other_Model"); + document.put("semantic_model", List.of(first, second)); + SalesforceBindings bindings = SalesforceBindings.fromString(""" + models: + Customer_Orders_Model: {dataspace: primary} + Other_Model: {dataspace: secondary} + """); + List result = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings) + .convert(YAML.writeValueAsString(document)); + assertEquals("primary", JSON.readTree(result.get(0)).get("dataspace").asText()); + assertEquals("secondary", JSON.readTree(result.get(1)).get("dataspace").asText()); + } + + private static void assertConversionError(String message, String bindings, String fixture) { + ConversionException error = assertThrows(ConversionException.class, + () -> output(SalesforceBindings.fromString(bindings), fixture)); + assertTrue(error.getMessage().contains(message), error.getMessage()); + } + + private static Map output(SalesforceBindings bindings, String fixture) throws Exception { + String json = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings).convert(fixture).get(0); + return JSON.readValue(json, new TypeReference<>() {}); + } + + private static String fixture() throws Exception { + return Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")); + } + + @SuppressWarnings("unchecked") + private static List> items(Map parent, String key) { + return (List>) parent.getOrDefault(key, List.of()); + } + + private static Map find(List> items, String name) { + return items.stream().filter(item -> name.equals(item.get("apiName"))).findFirst().orElseThrow(); + } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java new file mode 100644 index 00000000..d512adbc --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.ossie.exception.ConversionException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class SalesforceModelValidatorTest { + private final SalesforceModelValidator validator = new SalesforceModelValidator(); + + @Test + void acceptsCompleteDirectModelWithoutMutatingEitherSide() { + Map source = source(); + Map target = target(); + String originalSource = source.toString(); + String originalTarget = target.toString(); + assertDoesNotThrow(() -> validator.validate(source, target)); + assertEquals(originalSource, source.toString()); + assertEquals(originalTarget, target.toString()); + } + + @ParameterizedTest + @ValueSource(strings = {"semanticDataObjects", "semanticCalculatedMeasurements"}) + void rejectsMissingExportedEntities(String property) { + Map target = target(); + target.remove(property); + assertTrue(error(source(), target).contains("was not exported")); + } + + @Test + void rejectsMissingFieldAndRequiredPhysicalBinding() { + Map target = target(); + object(target).put("semanticMeasurements", List.of()); + assertTrue(error(source(), target).contains("orders.amount")); + target = target(); + field(target).remove("dataObjectFieldName"); + assertTrue(error(source(), target).contains("dataObjectFieldName")); + } + + @Test + void rejectsEquivalentSourceIdentifiersAndDuplicateTargetFields() { + Map source = source(); + sourceObject(source).put("fields", List.of(Map.of("name", "amount"), Map.of("name", "AMOUNT"))); + assertTrue(error(source, target()).contains("Duplicate")); + Map target = target(); + object(target).put("semanticDimensions", List.of(new LinkedHashMap<>(field(target)))); + assertTrue(error(source(), target).contains("Duplicate")); + } + + @Test + void rejectsModelLevelCalculatedNameCollisionAcrossDimensionsAndMetrics() { + Map target = target(); + target.put("semanticCalculatedDimensions", List.of(calculation("total", "1"))); + assertTrue(error(source(), target).contains("Duplicate model-level calculated field")); + } + + @ParameterizedTest + @ValueSource(strings = {"SUM([missing].[amount])", "SUM([orders].[missing])", "[missing]"}) + void rejectsDanglingFinalFormulaReferences(String expression) { + Map target = target(); + calculation(target).put("expression", expression); + String error = error(source(), target); + assertTrue(error.contains("references"), error); + assertTrue(error.contains("missing"), error); + } + + @ParameterizedTest + @ValueSource(strings = {"IF '[missing].[field]' = '[missing].[field]' THEN SUM([orders].[amount]) ELSE 0 END", + "IF \"[missing]\" = \"[missing]\" THEN SUM([orders].[amount]) ELSE 0 END", + "IF 'it''s [missing]' = 'it''s [missing]' THEN SUM([orders].[amount]) ELSE 0 END"}) + void bracketTextInsideStringsDoesNotBecomeAReference(String expression) { + Map target = target(); + calculation(target).put("expression", expression); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @ParameterizedTest + @ValueSource(strings = {"TABLEAU", "SQL", ""}) + void requiresTuaOnFinalCalculatedFields(String syntax) { + Map target = target(); + calculation(target).put("syntax", syntax); + assertTrue(error(source(), target).contains("requires syntax 'Tua'")); + } + + @Test + void detectsCyclesInNativeCalculatedReferences() { + Map target = target(); + calculation(target).put("expression", "[other]"); + target.put("semanticCalculatedDimensions", List.of(calculation("other", "[total]"))); + assertTrue(error(source(), target).contains("Cyclic")); + } + + @Test + void rejectsGenericHandlerResidueInsteadOfTreatingItAsNativeRelationship() { + Map target = target(); + target.put("semanticRelationships", List.of(Map.of("name", "raw", "from", "orders", "to", "orders", + "from_columns", List.of("amount"), "to_columns", List.of("amount")))); + assertTrue(error(source(), target).contains("apiName")); + } + + @Test + void finalMetricConnectivityUsesOnlyEnabledRelationships() { + Map source = source(); + Map target = target(); + datasets(source).add(new LinkedHashMap<>(Map.of("name", "returns", "fields", List.of(Map.of("name", "amount"))))); + objects(target).add(new LinkedHashMap<>(Map.of("apiName", "returns", "dataObjectName", "returns__dll", + "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c"))))); + calculation(target).put("expression", "SUM([orders].[amount]) - SUM([returns].[amount])"); + Map relationship = new LinkedHashMap<>(Map.of("apiName", "join", "leftSemanticDefinitionApiName", "orders", + "rightSemanticDefinitionApiName", "returns", "criteria", List.of(Map.of("leftSemanticFieldApiName", "amount", + "rightSemanticFieldApiName", "amount")), "cardinality", "ManyToMany", "joinType", "Auto", "isEnabled", false)); + target.put("semanticRelationships", List.of(relationship)); + assertTrue(error(source, target).contains("disconnected by enabled relationships")); + relationship.remove("isEnabled"); + assertTrue(error(source, target).contains("disconnected by enabled relationships")); + relationship.put("isEnabled", true); + assertDoesNotThrow(() -> validator.validate(source, target)); + } + + @Test + void acceptsOptionalNativeRelationshipMetadataWithoutAssumingAnEnabledEdge() { + Map source = source(); + source.remove("metrics"); + Map target = target(); + target.remove("semanticCalculatedMeasurements"); + target.put("semanticRelationships", List.of(Map.of("apiName", "native", "leftSemanticDefinitionApiName", "orders", + "rightSemanticDefinitionApiName", "orders", "criteria", List.of(Map.of("leftSemanticFieldApiName", "amount", + "rightSemanticFieldApiName", "amount"))))); + assertDoesNotThrow(() -> validator.validate(source, target)); + } + + @Test + void declaredPrimaryAndUniqueKeysAreCheckedWithoutInventingNativeSchemaFields() { + Map source = source(); + sourceObject(source).put("primary_key", List.of("amount")); + sourceObject(source).put("unique_keys", List.of(List.of("amount"))); + Map target = target(); + assertDoesNotThrow(() -> validator.validate(source, target)); + assertFalse(object(target).containsKey("primaryNameField")); + sourceObject(source).put("primary_key", List.of("missing")); + assertTrue(error(source, target).contains("key references unknown field")); + } + + @Test + void detectsChangedCompositeCorrespondenceAfterLaterExtensionRestoration() { + Map source = source(); + source.put("relationships", List.of(Map.of("name", "join", "from", "orders", "to", "orders", + "from_columns", List.of("amount"), "to_columns", List.of("amount")))); + Map target = target(); + object(target).put("semanticDimensions", List.of(Map.of("apiName", "other", "dataObjectFieldName", "other__c"))); + target.put("semanticRelationships", List.of(Map.of("apiName", "join", "leftSemanticDefinitionApiName", "orders", + "rightSemanticDefinitionApiName", "orders", "criteria", List.of(Map.of("leftSemanticFieldApiName", "other", + "rightSemanticFieldApiName", "amount")), "cardinality", "ManyToOne", "joinType", "Auto", "isEnabled", true))); + assertTrue(error(source, target).contains("changed join key correspondence")); + } + + @Test + void checksDerivedFieldCoverageUsingTheExistingPlan() { + Map source = source(); + sourceObject(source).put("source", "orders__dll"); + sourceObject(source).put("fields", List.of( + Map.of("name", "amount", "datatype", "Decimal", "expression", Map.of("dialects", + List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount__c")))), + Map.of("name", "derived", "datatype", "Decimal", "expression", Map.of("dialects", + List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount + 1")))))); + Map target = target(); + FieldExpressionPlan plan = new FieldExpressionPlan(source, target); + assertTrue(assertThrows(ConversionException.class, () -> validator.validate(source, target, plan)) + .getMessage().contains("orders.derived")); + target.put("semanticCalculatedDimensions", List.of(calculation(plan.calculatedApiName("orders", "derived"), + "[orders].[amount] + 1"))); + assertDoesNotThrow(() -> validator.validate(source, target, plan)); + } + + @Test + void validatesNativeDependencyMetadataAfterExtensionRestoration() { + Map target = target(); + calculation(target).put("dependencies", List.of(Map.of("dependentDefinitionApiName", "orders", + "dependentFieldApiName", "missing"))); + assertTrue(error(source(), target).contains("dependency references missing field")); + calculation(target).put("dependencies", List.of(Map.of("dependentDefinitionApiName", "orders", + "dependentFieldApiName", "amount"))); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @Test + void compilesNativeOnlyCalculationsInsteadOfTrustingMetadata() { + Map target = target(); + target.put("semanticCalculatedDimensions", List.of(calculation("external", "BOGUS(1)"))); + String failure = error(source(), target); + assertTrue(failure.contains("Native calculated field 'external'"), failure); + assertTrue(failure.contains("BOGUS"), failure); + Map external = calculation("external", "1"); + external.put("dataType", "Text"); + target.put("semanticCalculatedDimensions", List.of(external)); + assertTrue(error(source(), target).contains("conflicts with native dataType")); + } + + @Test + void nativeOnlyMixedAggregationCannotBypassTheSharedAnalyzer() { + Map target = target(); + field(target).put("dataType", "Number"); + target.put("semanticCalculatedMeasurements", List.of(calculation(target), + calculation("external", "SUM([orders].[amount]) + [orders].[amount]"))); + String failure = error(source(), target); + assertTrue(failure.contains("Native calculated field 'external'"), failure); + assertTrue(failure.contains("aggregate") || failure.contains("aggregat"), failure); + } + + @Test + void explicitlyMarkedNativeRowMeasurementsRemainSupported() { + Map target = target(); + field(target).put("dataType", "Number"); + Map external = calculation("external", "[orders].[amount] * 2"); + target.put("semanticCalculatedMeasurements", List.of(calculation(target), external)); + assertTrue(error(source(), target).contains("expression aggregation conflicts")); + external.put("level", "Row"); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @Test + void explicitlyMarkedNativeAggregateDimensionKeepsItsSupportedLevel() { + Map target = target(); + field(target).put("dataType", "Number"); + Map external = calculation("external", "SUM([orders].[amount]) > 0"); + external.put("dataType", "Boolean"); + target.put("semanticCalculatedDimensions", List.of(external)); + assertTrue(error(source(), target).contains("expression aggregation conflicts")); + external.put("level", "AggregateFunction"); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @Test + void nativeOnlyCalculationDependenciesUseFinalTypesInTopologicalOrder() { + Map target = target(); + field(target).put("dataType", "Number"); + target.put("semanticCalculatedDimensions", List.of(calculation("row", "[orders].[amount] + 1"))); + target.put("semanticCalculatedMeasurements", List.of(calculation(target), calculation("external", "SUM([row])"))); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @Test + void nullOnlyNativeFormulaRequiresADeclaredTypeWithoutCrashing() { + Map target = target(); + Map external = calculation("external", "NULL"); + target.put("semanticCalculatedDimensions", List.of(external)); + assertTrue(error(source(), target).contains("dataType")); + external.put("dataType", "Text"); + assertDoesNotThrow(() -> validator.validate(source(), target)); + } + + @Test + void nativeArraysCannotDisappearBehindAlreadyPresentGeneratedArrays() { + Map source = source(); + source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"semanticCalculatedMeasurements\":[{\"apiName\":\"external\",\"expression\":\"1\",\"syntax\":\"Tua\"}]}"))); + assertTrue(error(source, target()).contains("Native extension semanticCalculatedMeasurements entity 'external' was not exported")); + Map target = target(); + target.put("semanticCalculatedMeasurements", List.of(calculation(target), calculation("external", "1"))); + assertDoesNotThrow(() -> validator.validate(source, target)); + // Matching core identities remain authoritative even when their native snapshot is stale. + source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"semanticCalculatedMeasurements\":[{\"apiName\":\"total\",\"expression\":\"STALE()\"}]}"))); + assertDoesNotThrow(() -> validator.validate(source, target())); + } + + @Test + void publicConverterRejectsUncompiledFunctionRestoredAtModelLevel() throws Exception { + String input = conversionInput("semanticCalculatedDimensions", + Map.of("apiName", "native_extra", "expression", "BOGUS(1)", "syntax", "Tua")); + ConversionException error = assertThrows(ConversionException.class, + () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); + assertTrue(error.getMessage().contains("native_extra"), error.getMessage()); + assertTrue(error.getMessage().contains("BOGUS"), error.getMessage()); + } + + @Test + void publicConverterRejectsNativeArrayLossAlongsideCoreMetrics() throws Exception { + String input = conversionInput("semanticCalculatedMeasurements", + Map.of("apiName", "native_extra", "expression", "1", "syntax", "Tua")); + ConversionException error = assertThrows(ConversionException.class, + () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); + assertTrue(error.getMessage().contains("native_extra"), error.getMessage()); + assertTrue(error.getMessage().contains("was not exported"), error.getMessage()); + } + + private static String conversionInput(String nativeArray, Map nativeCalculation) throws Exception { + com.fasterxml.jackson.databind.ObjectMapper json = new com.fasterxml.jackson.databind.ObjectMapper(); + Map source = source(); + sourceObject(source).put("source", "orders__dll"); + sourceObject(source).put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + "{\"dataObjectType\":\"Dlo\"}"))); + sourceObject(source).put("fields", List.of(Map.of("name", "amount", "datatype", "Decimal", "expression", + Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount__c")))))); + source.put("metrics", List.of(Map.of("name", "total", "datatype", "Decimal", "expression", + Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "SUM(orders.amount)")))))); + source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", + json.writeValueAsString(Map.of("dataspace", "default", nativeArray, List.of(nativeCalculation)))))); + return json.writeValueAsString(Map.of("version", "0.2.0.dev0", "semantic_model", List.of(source))); + } + + private String error(Map source, Map target) { + return assertThrows(ConversionException.class, () -> validator.validate(source, target)).getMessage(); + } + + private static Map source() { + Map source = new LinkedHashMap<>(); + source.put("name", "sales"); + source.put("datasets", new ArrayList<>(List.of(new LinkedHashMap<>(Map.of("name", "orders", "fields", List.of(Map.of("name", "amount"))))))); + source.put("metrics", List.of(Map.of("name", "total"))); + return source; + } + + private static Map target() { + Map target = new LinkedHashMap<>(); + target.put("apiName", "sales"); + target.put("semanticDataObjects", new ArrayList<>(List.of(new LinkedHashMap<>(Map.of("apiName", "orders", "dataObjectName", "orders__dll", + "semanticMeasurements", List.of(new LinkedHashMap<>(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c")))))))); + target.put("semanticCalculatedMeasurements", List.of(calculation("total", "SUM([orders].[amount])"))); + return target; + } + + private static Map calculation(String name, String expression) { + return new LinkedHashMap<>(Map.of("apiName", name, "expression", expression, "syntax", "Tua")); + } + + @SuppressWarnings("unchecked") + private static List> datasets(Map source) { return (List>) source.get("datasets"); } + @SuppressWarnings("unchecked") + private static List> objects(Map target) { return (List>) target.get("semanticDataObjects"); } + private static Map sourceObject(Map source) { return datasets(source).get(0); } + private static Map object(Map target) { return objects(target).get(0); } + @SuppressWarnings("unchecked") + private static Map field(Map target) { return ((List>) object(target).get("semanticMeasurements")).get(0); } + @SuppressWarnings("unchecked") + private static Map calculation(Map target) { return ((List>) target.get("semanticCalculatedMeasurements")).get(0); } +} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java new file mode 100644 index 00000000..3166d641 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.ossie.exception.InvalidInputException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class SourceDocumentValidationTest { + @TempDir Path directory; + + @Test + void rejectsASecondOsiYamlDocumentRatherThanOmittingItsModel() throws Exception { + String valid = fixture("ossieToSalesforce.yaml"); + String second = valid.replace("Customer_Orders_Model", "Other_Model"); + assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, valid + "\n---\n" + second, "Trailing token"); + } + + @Test + void rejectsASecondNativeJsonObjectRatherThanOmittingIt() throws Exception { + String valid = fixture("salesforceToOssie.json"); + assertInvalid(ConversionDirection.SALESFORCE_TO_OSSIE, valid + "\n" + valid, "Trailing token"); + } + + @Test + void rejectsDuplicateYamlModelPropertiesBeforeTheyOverwriteTheFirstValue() throws Exception { + String input = fixture("ossieToSalesforce.yaml").replace(" - name: Customer_Orders_Model", + " - name: Customer_Orders_Model\n name: Silent_Replacement"); + assertTrue(input.contains("Silent_Replacement")); + assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, input, "Duplicate field 'name'"); + } + + @Test + void rejectsDuplicateNestedYamlTypesBeforeTheyChangeFieldMeaning() throws Exception { + String input = fixture("ossieToSalesforce.yaml").replaceFirst("(?m)^([ ]*)datatype: String$", + "$1datatype: String\n$1datatype: Integer"); + assertTrue(input.contains("datatype: Integer")); + assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, input, "Duplicate field 'datatype'"); + } + + @Test + void rejectsDuplicateNativeJsonPropertiesBeforeTheyOverwriteTheFirstValue() throws Exception { + String valid = fixture("salesforceToOssie.json"); + String input = "{\"apiName\":\"Silently_Replaced\"," + valid.substring(valid.indexOf('{') + 1); + assertInvalid(ConversionDirection.SALESFORCE_TO_OSSIE, input, "Duplicate field 'apiName'"); + } + + @Test + void fileApiWritesNothingForTrailingSourceDocumentsAndAcceptsSingleDocumentComments() throws Exception { + String valid = fixture("ossieToSalesforce.yaml"); + Converter converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); + assertEquals(1, converter.convert(valid + "\n# trailing comments are part of the same document\n").size()); + Path input = directory.resolve("input.yaml"); + Files.writeString(input, valid + "\n---\nnull\n"); + Path existing = directory.resolve("Customer_Orders_Model.json"); + Files.writeString(existing, "preserve existing output"); + assertThrows(InvalidInputException.class, () -> converter.convert(input, directory)); + assertEquals("preserve existing output", Files.readString(existing)); + try (var files = Files.list(directory)) { assertEquals(2, files.count()); } + } + + private static void assertInvalid(ConversionDirection direction, String content, String message) { + InvalidInputException error = assertThrows(InvalidInputException.class, + () -> ConverterFactory.getConverter(direction).convert(content)); + assertTrue(error.getMessage().contains(message), error.getMessage()); + } + + private static String fixture(String file) throws Exception { + return Files.readString(Path.of("src/test/resources/examples", file)); + } +} diff --git a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml index 101588f0..df53a92b 100644 --- a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml +++ b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml @@ -357,35 +357,6 @@ semantic_model: - product_id to_columns: - product_id - - name: Customers_ByDomain - from: Customers - to: Orders - from_columns: - - customer_email_domain - to_columns: - - order_id - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Invalid Relationship - Uses Calculated Field" - } - - name: Orders_ByYear - from: Orders - to: Products - from_columns: - - order_year - to_columns: - - product_id - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Invalid Relationship - Uses Calculated Field", - "cardinality" : "ManyToMany", - "joinType" : "Auto", - "isEnabled" : true - } metrics: - description: Sum of all order amounts name: total_revenue From 6ee2e4558cd06322764b2da4b3bd21cef47f0571 Mon Sep 17 00:00:00 2001 From: Saurabh Deshpande <43935865+saurabhdeshp@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:07:27 -0700 Subject: [PATCH 5/6] refactor(salesforce): keep metric conversion focused on issue 403 Defer physical bindings, derived fields, metric dependencies, extension changes and whole-model validation. Keep JSqlParser-backed metric parsing, typed Tua generation and targeted field/relationship reference checks. Consolidate the compiler around metric-specific components and retain focused semantic and integration regressions. Validation: 254 tests passed with no skips, Apache RAT passed, and the packaged CLI export smoke check passed. --- converters/salesforce/README.md | 368 +++++------- converters/salesforce/pom.xml | 1 - .../ossie/app/OssieSalesforceConverter.java | 24 +- .../ossie/converter/AbstractConverter.java | 6 - .../ossie/converter/ConversionContext.java | 39 -- .../ossie/converter/ConverterFactory.java | 5 - .../apache/ossie/converter/ConverterImpl.java | 44 +- .../converter/CustomExtensionHandler.java | 49 +- .../ossie/converter/ExpressionAnalyzer.java | 227 ------- .../ossie/converter/ExpressionCompiler.java | 98 --- .../converter/ExpressionFunctionRegistry.java | 76 --- .../ossie/converter/ExpressionTokens.java | 112 ---- .../ossie/converter/FieldExpressionPlan.java | 385 ------------ .../ossie/converter/FieldMappingHandler.java | 340 ++++++++--- .../converter/MetricCompilationPlan.java | 96 --- ...pressionAst.java => MetricExpression.java} | 13 +- .../converter/MetricExpressionTranslator.java | 309 +++++++++- .../ossie/converter/MetricFieldResolver.java | 352 ++++++----- .../ossie/converter/MetricMappingHandler.java | 34 +- .../converter/RelationshipMappingHandler.java | 122 +++- .../ossie/converter/SalesforceBindings.java | 174 ------ .../converter/SalesforceModelValidator.java | 567 ------------------ ...er.java => SqlMetricExpressionParser.java} | 24 +- .../ossie/converter/TuaExpressionEmitter.java | 83 --- ...er.java => TuaMetricExpressionParser.java} | 101 +++- .../converter/pipeline/PipelineStep.java | 6 - .../java/org/apache/ossie/MetricCliTest.java | 155 ----- .../ossie/MetricExportIntegrationTest.java | 141 ++--- .../ossie/OssieToSalesforceConverterTest.java | 24 +- .../ossie/SalesforceToOssieConverterTest.java | 4 +- .../converter/ConstantFieldMetricTest.java | 148 ----- .../converter/CustomExtensionHandlerTest.java | 120 ---- .../converter/ExpressionCompilerTest.java | 194 ------ .../converter/FieldExpressionPlanTest.java | 346 ----------- .../converter/MetricCompilationPlanTest.java | 133 ---- .../MetricExpressionSemanticsTest.java | 135 +---- .../MetricExpressionTranslatorTest.java | 70 ++- .../converter/MetricFieldResolverTest.java | 103 +++- .../RelationshipMappingHandlerTest.java | 257 -------- .../converter/SalesforceBindingsTest.java | 221 ------- .../SalesforceModelValidatorTest.java | 362 ----------- .../SourceDocumentValidationTest.java | 92 --- .../resources/examples/ossieToSalesforce.yaml | 29 + 43 files changed, 1416 insertions(+), 4773 deletions(-) delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java rename converters/salesforce/src/main/java/org/apache/ossie/converter/{ExpressionAst.java => MetricExpression.java} (84%) delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java rename converters/salesforce/src/main/java/org/apache/ossie/converter/{SqlExpressionParser.java => SqlMetricExpressionParser.java} (90%) delete mode 100644 converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java rename converters/salesforce/src/main/java/org/apache/ossie/converter/{TuaExpressionParser.java => TuaMetricExpressionParser.java} (52%) delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java delete mode 100644 converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md index af3880e7..77e81464 100644 --- a/converters/salesforce/README.md +++ b/converters/salesforce/README.md @@ -22,10 +22,8 @@ A two-way converter between [Ossie semantic models](../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html). This converter supports conversion in both directions between Ossie YAML and -Salesforce Semantic Model JSON. Each input must contain one document; duplicate -mapping keys and trailing documents are rejected before conversion. Supported -unmapped Salesforce properties are preserved in `custom_extensions`; see the -mapping reference for direction-specific limits. +Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in +`custom_extensions`; see the mapping reference for direction-specific limits. ## Requirements @@ -44,8 +42,7 @@ This produces a self-contained executable jar at `target/ossie-salesforce-conver ## Setup -Both conversion directions validate input and output. Obtain the Salesforce schema -before building so it is bundled into the jar; conversion fails if it is missing. +Obtain the Salesforce schema before building so it is bundled into the jar. Maven copies the canonical Ossie schema from `../../core-spec/ossie-schema.json`. ### Salesforce Semantic Model Schema @@ -60,10 +57,9 @@ Run the complete suite, including Salesforce schema checks, with: mvn -DrequireSalesforceSchema=true clean verify ``` -The property explicitly fails the suite when the Salesforce schema is missing, -including tests that otherwise skip for missing resources. Public API and CLI -checks require both schemas. `verify` also checks Apache license headers. Do not -commit downloaded schemas. +The property makes a missing Salesforce schema fail the test run. Without it, +schema-dependent tests retain their existing skip behavior. `verify` also checks +Apache license headers. Do not commit downloaded schemas. ## Usage @@ -138,8 +134,8 @@ ossieToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); ### Features -- **Schema-validated** - Input and final output are validated against JSON Schema -- **Explicit conversion boundaries** - Supported native metadata is preserved; unsupported expressions, missing references and omitted declared entities fail conversion +- **Schema-validated** - Input is validated against JSON Schema before processing +- **Lossless conversion** - Unmapped properties are preserved in `custom_extensions` - **Bidirectional** - Supports both directions, with direction-specific limits documented below - **Supports Ossie Specification v0.2.0.dev0** @@ -172,7 +168,7 @@ ossieToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/")); | `datasets[].name` | `semanticDataObjects[].apiName` | | `datasets[].source` | `semanticDataObjects[].dataObjectName` | | Direct `fields[]` | Split into `semanticDimensions[]` and `semanticMeasurements[]` based on `dimension` presence | -| Derived row fields | Validated Tua in `semanticCalculatedDimensions[]`, with dataset-qualified generated names | +| Calculated Tableau fields | `semanticCalculatedDimensions[]` through the existing expression-analysis path | | `expression.dialects[].expression` | `dataObjectFieldName` | | Field `datatype` | Field `dataType` when a safe mapping exists | | `relationships[]` | `semanticRelationships[]` | @@ -211,22 +207,22 @@ type exists: | `Boolean` | `Boolean` | | `Date` | `Date` | | `DateTime`, `DateTimeTz` | `DateTime` | -| `Time`, `Opaque` | Rejected unless a compatible exact native extension type exists | +| `Time`, `Opaque` | Omitted with a warning unless an exact extension type exists | Salesforce has one `DateTime` type, so exporting timezone-free Ossie `DateTime` loses its distinction from `DateTimeTz`; the converter logs a warning because a subsequent Salesforce import interprets that value as `DateTimeTz`. An exact Salesforce extension value takes precedence over the portable mapping. -If it conflicts with `datatype`, conversion fails. An absent direct-field datatype -can remain unspecified in metadata, but an expression using it needs a known, -compatible type. +If it conflicts with `datatype`, the converter preserves the exact Salesforce +value and logs a warning. ### Field Role and Time Dimensions `datatype` does not determine whether an Ossie field is a dimension or a fact. For direct fields, the presence of the `dimension` object determines whether the -field is exported to `semanticDimensions` or `semanticMeasurements`. A derived row expression becomes a model-level calculated dimension. +field is exported to `semanticDimensions` or `semanticMeasurements`. A calculated +Tableau expression follows the converter's existing calculated-dimension path. On import, Salesforce `Date` and `DateTime` dimensions set `dimension.is_time` to `true`; other dimension types set it to `false`. On export, `dimension.is_time` @@ -234,78 +230,38 @@ does not invent or override a scalar type. This preserves Ossie's separation of logical data type from temporal role, including integer year and string month dimensions. -### Relationships +### Relationship Handling -Every declared relationship must be exported with the same endpoint and ordered -join-key pairs. Missing fields, unequal composite-key lengths, duplicate names, -and calculated join keys fail conversion; no relationship is silently filtered. -The default cardinality is `ManyToOne`, following OSI. Valid explicit Salesforce -cardinality metadata is preserved for native round trips. Declared primary and -unique keys are checked against references and the unique side of the chosen -cardinality. This checks metadata consistency, not uniqueness in actual data. - -On Salesforce import, unsupported Formula/SemanticField relationships remain in -model extensions. Export currently rejects those joins explicitly. Core -relationships and native extension arrays must not cause one another to disappear. -An extension-only relationship with omitted enablement metadata cannot establish -proven connectivity for a cross-dataset calculation. - -### Field expressions and physical bindings - -Field conversion parses the selected expression dialect (`TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`) and builds one dependency plan per model. Every declared field must either produce a direct field or a supported calculated dimension. Unsupported expressions fail with the dataset and field name; they are never silently omitted. - -A direct SQL identifier defines a physical column. For example, a field named `amount` with expression `revenue__c` emits `apiName: amount` and `dataObjectFieldName: revenue__c`. Double-quoted identifiers preserve their contents, including spaces, operators, periods, and escaped quotes. A verified qualification such as `warehouse.sales.orders.revenue__c` is reduced to its column name. Qualifiers must match the declared dataset name or its physical source; the converter does not guess unrelated catalog paths. - -A derived SQL expression can reference declared fields or physical columns exposed by direct fields in that same dataset. For example, with direct fields `amount = revenue__c` and `cost = cost__c`, both `amount - cost` and `revenue__c - cost__c` bind to `[Orders].[amount] - [Orders].[cost]`. A name matching different physical columns or semantic fields is rejected as ambiguous. SQL unquoted identifiers use case folding; quoted identifiers preserve case. A single-identifier SQL expression always defines a physical binding, even when it matches another semantic field name. A native `TABLEAU` expression such as `[Orders].[amount]` instead denotes a semantic alias and is emitted as a calculated dimension. - -Derived fields can depend on other derived fields, including declarations appearing later in the input. The plan expands those dependencies into row expressions over direct semantic fields. This lets metrics consume derived fields without relying on an undocumented global calculated-field reference syntax. Each calculated dimension uses `syntax: Tua`, a stable name based on `dataset__field`, and flattened direct-field dependencies. Name collisions, including collisions with metric names and sanitized names from other datasets, receive deterministic hash suffixes. - -Row fields must stay within their dataset and cannot contain aggregations. -A constant-only row field can be emitted as a calculated dimension, but a metric -cannot reference it until a native dataset anchor is supported; otherwise -`SUM(dataset.constant_one)` would collapse to an unscoped `SUM(1)`. Put aggregate expressions in `metrics`. Referenced fields need supported, compatible datatypes; inferred result types must agree with declared types. Unknown references, dependency cycles, unsupported operations, and incompatible types stop conversion. Dependency chains are limited to 128 fields, and the shared expression emitter limits expanded formulas to 131,072 characters. These are converter resource limits, not advertised Tableau platform limits. - -Environment-specific Salesforce bindings are applied after expressions have been bound in their original source scope. They can change the target data object and direct physical column names while preserving semantic API names, formulas, dependency identities, and the OSI input. Thus rebinding `amount` to `NetRevenue__c` still leaves its formulas referring to `[Orders].[amount]`. - -On Salesforce import, direct `dataObjectFieldName` values are represented as quoted `ANSI_SQL` identifiers. Calculated expressions remain `TABLEAU`. This preserves punctuation and avoids mislabeling physical column names as native Tableau formulas. - -These checks establish local binding and supported translation behavior. Native formula validation and result equivalence still require validation against the intended Tableau Next environment and dataset. +**Unsupported relationships** (containing Formula or SemanticField types) are stored in `custom_extensions` at the model level rather than being converted to Ossie relationships. ### Metric expressions -Fields and metrics select `TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`, independent -of entry order. Duplicate dialect entries, an empty selected expression or an -unsupported selected expression fail without falling back to another dialect. -The source OSI model needs no new dialect entries or formula edits. +Metrics select `TABLEAU`, then `SNOWFLAKE`, then `ANSI_SQL`, independent of entry +order. The selected expression is parsed and validated; an invalid preferred +expression fails rather than falling back to another dialect. Duplicate selected +dialect entries are errors. Successful conversion exports every declared metric. The target is the Salesforce/Tableau Next semantic model's [Tua calculation language](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-functions.html). -Calculated measurements emit `syntax: Tua`, `aggregationType: UserAgg` and -`level: AggregateFunction`. The default numeric `dataType` is `Number`; compatible -native `Currency`/`Percentage` and display metadata such as labels and decimal -places are retained. Stale extension expressions cannot replace compiled formulas. Salesforce -extension data must be a single JSON object with unique keys; invalid metadata -fails with the owning entity name. -See [calculated fields](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-calculated-fields.html) +Calculated measurements emit `syntax: Tua`, `dataType: Number`, and +`aggregationType: UserAgg`, so an already aggregated formula is not aggregated +again. See [calculated fields](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-calculated-fields.html) and [aggregation rules](https://developer.salesforce.com/docs/data/semantic-layer/guide/query-api-in-depth-aggregation.html). -| SQL input | Tua output / restriction | -|-----------|--------------------------| +| SQL input | Tua output | +|-----------|------------| | `SUM`, `AVG`, `MIN`, `MAX`, `COUNT(field)` | Same aggregate | | `COUNT(DISTINCT field)` | `COUNTD(field)` | | `+`, `-`, `*`, `/`, parentheses, numeric constants | Explicitly grouped arithmetic | | Searched `CASE WHEN` | `IF … THEN … ELSEIF … ELSE … END` | | Comparisons, `AND`, `OR`, `NOT` | Equivalent grouped operators | -| `COALESCE(a, b, …)` | Nested two-argument `IFNULL` | +| `COALESCE(a, b, …)` | Nested `IFNULL` | | `NULLIF(a, b)` | `IF a = b THEN NULL ELSE a END` | | `IS NULL`, `IS NOT NULL` | `ISNULL`, `NOT ISNULL` | | `ABS`, `ROUND`, `CEIL`, `FLOOR` | `ABS`, `ROUND`, `CEILING`, `FLOOR` | -| `YEAR(date)` | `YEAR`; requires `Date` or timezone-free `DateTime` | -| `LENGTH(text)` | `LEN` | -| `POSITION(needle IN text)` | `FIND(text, needle)` | -| `SUBSTRING(text, start[, length])` | `MID`; start must be provably positive and optional length nonnegative | -These constructs compose. For example: +These constructs compose. For example, with declared numeric fields `profit` and +`revenue` in dataset `orders`: ```yaml metrics: @@ -314,170 +270,146 @@ metrics: expression: dialects: - dialect: SNOWFLAKE - expression: total_profit / NULLIF(total_revenue, 0) - - name: total_profit - datatype: Decimal - expression: - dialects: - - dialect: SNOWFLAKE - expression: SUM(orders.profit) - - name: total_revenue - datatype: Decimal - expression: - dialects: - - dialect: SNOWFLAKE - expression: SUM(orders.revenue) + expression: SUM(orders.profit) / NULLIF(SUM(orders.revenue), 0) ``` -Named metric references resolve independently of declaration order, are checked -for cycles and are inlined as aggregate expressions. Compiled dependencies are -cached per model. `[metric]` is the corresponding native `TABLEAU` reference. -A field and metric sharing an unqualified name are ambiguous; qualify the field -or use a unique metric name. Aggregating an already aggregated metric fails. +The resulting expression is: + +```text +(SUM([orders].[profit]) / (IF (SUM([orders].[revenue]) = 0) THEN NULL ELSE SUM([orders].[revenue]) END)) +``` -**Binding and types.** Metric references use declared logical field names, -including supported derived fields. Physical column names and source paths are -not metric aliases. Unqualified SQL fields must be unique across datasets. -Regular SQL names normalize to uppercase; double-quoted names match the normalized +**Binding and types.** References resolve against declared dataset and field +names, then against fields actually emitted to Salesforce. Physical column names +and source paths are not aliases. Unqualified SQL fields must be unique. Regular +SQL names normalize to uppercase; double-quoted names match the normalized declaration exactly, following the [expression specification](../../core-spec/expression_language.md). Thus `"ORDERS"."AMOUNT"` matches regular declarations `orders.amount`, while -`"orders"."amount"` requires explicitly quoted lowercase declarations. Native -`TABLEAU` uses exact `[dataset].[field]` names. Legacy complete bracket references -in `ANSI_SQL` are retained as a compatibility case; Snowflake requires SQL quotes. +`"orders"."amount"` requires explicitly quoted lowercase declarations. Target +API names are preserved; conversion does not rename fields or discover columns. Names containing brackets or control characters fail because their Tua escaping -is not established. - -Referenced fields need known compatible datatypes. Arithmetic and `SUM`/`AVG` -require numbers; branches and comparisons require compatible operand types. -Metrics must return numbers and be aggregated or constant. Declared `Integer` -results cannot conceal fractional expressions. `CEIL`, `FLOOR` and `ROUND` at -nonpositive precision infer integral values. All-null metrics need an explicit -numeric datatype. Mixed row/aggregate expressions and nested aggregates fail. - -One aggregate cannot combine fields from different datasets. Separate aggregates -may reference datasets connected through enabled exported relationships, including -references introduced by metric dependencies. Connectivity checks do not prove -that the target query planner preserves the intended grain. - -**Boundaries.** Parser acceptance never implies target support. Windows, LOD, -subqueries, SQL casts, simple `CASE`, UDFs, `COUNT(*)`, comments, backslash string -escapes, implicit type coercions and unlisted functions are unsupported. No raw -SQL fallback is emitted. `COUNT`/`COUNTD` accept declared fields. `ROUND` accepts -one argument or an integer-literal precision; rounding-mode overloads are rejected. -Literal zero divisors fail; `NULLIF` can guard a denominator. Each expression is -bounded to 32,768 input characters, 8,192 tokens, 128 syntax/dependency levels and -131,072 generated characters. These are converter resource limits, not platform -limits. Errors identify the field/metric and rejected construct or reference. - -### Environment bindings - -Use an optional JSON or YAML manifest to bind the same OSI model to a Salesforce -environment. This file contains deployment names, never business formulas: - -```yaml -models: - sales: - dataspace: default - datasets: - orders: - dataObjectName: Orders__dll - dataObjectType: Dlo - fields: - profit: NetProfit__c - revenue: Revenue__c -``` - -Only listed values are overridden. Model, dataset and field keys are exact OSI -names. Fields must be direct physical bindings. Unknown names/properties, -duplicate keys, multiple YAML documents and bindings for calculated fields fail. -Native object types must match the bundled Salesforce schema; this is not catalog -discovery or DLO/DMO provisioning. Semantic names, formulas and the OSI file remain -unchanged. Without a manifest, existing source and extension mappings apply. - -```bash -java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar \ - toSF input.yaml --bindings bindings.yaml -``` - -```java -SalesforceBindings bindings = SalesforceBindings.fromPath(Path.of("bindings.yaml")); -Converter converter = ConverterFactory.getConverter( - ConversionDirection.OSSIE_TO_SALESFORCE, bindings); -List output = converter.convert(osiYaml); -``` - -Bindings are export-only. CLI usage/input errors exit with code 1/2; conversion -and schema errors are printed to stderr with exit code 3. All models are converted -and validated before the file API begins writing any output, so conversion failure -in a later model does not leave earlier model files behind. +is not established. Referenced fields must also have a single, unqualified physical +column binding in the exported model. This rejects derived expressions such as +`profit+tax` that the existing field mapper can misclassify as physical columns. +Derived-field compilation and normalization of qualified/quoted physical bindings +belong to separate converter work. + +`TABLEAU` references use exact `[dataset].[field]` API names. Its supported formula +subset is the Tua equivalents above, including `IF`, `IFNULL`, `ISNULL`, `COUNTD` +and `CEILING`. Existing `ANSI_SQL` expressions using complete bracket notation +retain that spelling as a compatibility case and receive the same validation. +Bracket notation is not accepted as Snowflake SQL. + +Fields need a known compatible datatype, either declared in OSI or restored from +an existing Salesforce field type. Arithmetic and `SUM`/`AVG` require numbers; +`MIN`/`MAX` also permit text and temporal values inside numeric calculations. +Comparisons and conditional/null-handling branches must have compatible types. +Metrics must return numbers; a declared `Integer` result cannot conceal a +fractional expression. Missing metric types are inferred. A formula must be +aggregated or constant: mixed row/aggregate expressions, nested aggregates, and +aggregates without a dataset field fail. A single aggregate cannot combine fields +from multiple datasets. Separate aggregates can use datasets connected through +explicitly enabled exported relationships. Each usable edge must match one source +relationship by name, endpoints and ordered join-key pairs; join fields must resolve +as direct fields. Missing or corrupted edges cannot establish connectivity. These +checks protect metrics without changing the relationship mapper or proving join grain. + +**Limits and compatibility.** `COUNT`/`COUNTD` take a field. `ROUND` supports one +argument or a second integer-literal precision; rounding-mode overloads are not +supported. `CEIL`/`FLOOR` take one argument. They and `ROUND` at zero or negative +precision infer integral values, so compatible `Integer` metrics are accepted. Simple `CASE`, date/time and string +functions, casts, metric/calculated-field references, windows, LOD, `COUNT(*)`, +SQL comments and backslash string escapes are outside this subset. String and +Boolean literals are supported in predicates. No null-to-zero setting or implicit +cast is added. Errors name the metric and explain the rejected construct or +reference. Literal zero divisors fail; use `NULLIF` to make a zero denominator +nullable. Expressions have bounded size, nesting and generated output. + +This replaces the unvalidated SQL fallback introduced in +[#402](https://github.com/apache/ossie/pull/402). Previously accepted invalid, +unsupported or untyped formulas now fail, including invalid `TABLEAU` input. +Metric/model extension preservation remains owned by #402. CLI conversion errors +are printed to stderr with exit code 3; this small change overlaps the conversion +error handling in [#286](https://github.com/apache/ossie/pull/286). + +**Implementation choice.** The existing mapping pipeline calls a focused Java +metric compiler. JSqlParser 5.3 parses SQL; a bounded Tua frontend handles native +formulas. Both produce a small immutable metric AST. Type/aggregation checks finish +before the private emitter writes Tua. No Python runtime is needed. JSqlParser is +used under its Apache-2.0 option; its unused JMH benchmark dependency is excluded. + +The implementation has five components: `SqlMetricExpressionParser`, +`TuaMetricExpressionParser`, `MetricExpression`, `MetricExpressionTranslator` and +`MetricFieldResolver`. The resolver caches successful bindings and verified graph +reachability within one model. To add supported syntax, update the relevant +frontend and metric rule with composition and semantic tests. There is no generic +model-planning or deployment framework in this change. + +**Validation boundary.** Unit tests cover parsing, binding and failure behavior. +Independent local Tua evaluation tests use synthetic rows with nulls, duplicates, +empty inputs and zero denominators. Those tests model the intended semantics; +they are not native Tableau Next execution. The published Salesforce **output** +schema checks structure, not formula syntax, catalog bindings or authoring API +acceptance. Before deployment, validate authoring and native queries in a test +org, including numeric precision, rounding ties, empty groups, null behavior and +unguarded dynamic division and multi-dataset grain. This converter does not provision Data 360 bindings or enrich +missing fields. ## Architecture -```text -OSI input -> source schema validation -> ConversionContext (one per model) - -> dataset mapping - -> FieldExpressionPlan: classify physical fields, bind and compile derived fields - -> validated relationships - -> MetricCompilationPlan: resolve fields/metrics, detect cycles, compile dependencies - -> native metadata restoration - -> physical environment bindings - -> final identity/reference/coverage checks -> target schema validation -> output - -ExpressionCompiler: - SQL -> JSqlParser frontend --+ - +-> immutable AST -> typed/aggregation analysis -> Tua emitter - TABLEAU -> bounded frontend -+ +``` + ┌───────────────────────┐ + │ OssieSalesforceConverter│ + │ (CLI App) │ + └───────────┬───────────┘ + │ + ┌───────┴────────┐ + │ ConverterFactory│ + └───────┬────────┘ + │ + ┌─────────────┴─────────────┐ + │ ConverterImpl │ + │ (Pipeline-based) │ + │ │ + │ • Configurable pipeline │ + │ • Bidirectional mapping │ + └─────────────┬─────────────┘ + │ + ┌─────────────┴─────────────┐ + │ Pipeline Handlers │ + ├───────────────────────────┤ + │ • DatasetMappingHandler │ + │ • FieldMappingHandler │ + │ • RelationshipHandler │ + │ • MetricMappingHandler │ + │ • SemanticModelHandler │ + └─────────────┬─────────────┘ + │ + ┌─────────────┴─────────────┐ + │ Support Components │ + ├───────────────────────────┤ + │ • GenericMappingEngine │ + │ • CustomExtensionHandler │ + │ • SchemaValidator │ + └───────────────────────────┘ ``` -JSqlParser 5.3 is used under its Apache-2.0 option for SQL syntax parsing. It does -not provide Tua semantics. `ExpressionFunctionRegistry` defines supported -dialect/function/arity combinations; `ExpressionAnalyzer` checks types and -aggregation; `TuaExpressionEmitter` writes only validated nodes. Supporting a new -function requires an explicit registry entry, semantic checks, lowering rule and -tests. SQLGlot is not required at runtime, and no Python bridge is introduced. - -`MetricFieldResolver` builds model-scoped identity and relationship indexes once. -Field and metric dependency plans cache compilation while enforcing depth and -output limits. No model state is stored in reusable handlers. The existing generic -mapping pipeline, schema resources, datatype mapper and public converter APIs remain -in use. `PipelineStep` retains the original map-based method for custom handlers. +**ConverterFactory** — Creates converter instances for specified direction -Final validation detects dangling references, duplicate identities, omitted core -or native-extension entities, disconnected calculations and inconsistent join keys. -Extension-only calculations also pass the shared expression compiler. Schema -validation runs after all extensions and bindings; neither can bypass the final -checks. A native extension array that conflicts with an emitted core array fails -instead of silently losing distinct entities. +**Pipeline Configuration** — Handlers and direction-specific settings defined in `ossie-salesforce-converter-config.yaml` -## Validation +**GenericMappingEngine** — Path-based property mapping using `mappings.yaml` configuration -```bash -mvn -DrequireSalesforceSchema=true clean verify -``` +**CustomExtensionHandler** — Preserves unmapped Salesforce properties in Ossie's `custom_extensions` for lossless bi-directional conversion -The suite covers parser/AST rejection, quoted names, types, aggregation, field and -metric dependency graphs, binding overlays, relationship preservation, native -metadata, CLI failures and both conversion directions. An independent local Tua -evaluator checks results over synthetic rows including nulls, duplicates, empty -inputs, decimals, dates and string fields. Stress cases exercise dependency depth, -expansion bounds and many metrics sharing one dependency. `verify` also runs Apache -RAT license-header checks. - -These are local checks, not native Tableau Next execution. The Salesforce output -schema validates structure, not tenant catalog existence or backend formula -semantics. Deployment still needs native formula/authoring validation and result -comparison in the intended org, especially numeric precision, rounding ties, nulls, -empty groups, timezones and multi-dataset grain. General Snowflake/ANSI SQL cannot -be promised equivalent where Tua has no supported construct. Unsupported cases -must use an explicit new lowering or a separately designed native execution route. +**SchemaValidator** — Validates input against JSON schemas before conversion ## Examples -- `src/test/resources/examples/ossieToSalesforce.yaml`: valid OSI export fixture -- `src/test/java/org/apache/ossie/MetricExportIntegrationTest.java`: public API, dependencies and bindings -- `src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java`: supported/rejected expressions -- `src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java`: derived fields and scope -- `src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java`: reverse conversion +See the test suite for sample models demonstrating various features: +- `src/test/resources/examples/ossieToSalesforce.yaml` - Ossie model example +- `src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java` - Ossie to Salesforce conversion tests +- `src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java` - Salesforce to Ossie conversion tests ## License diff --git a/converters/salesforce/pom.xml b/converters/salesforce/pom.xml index ff3a4f83..2bb0d20b 100644 --- a/converters/salesforce/pom.xml +++ b/converters/salesforce/pom.xml @@ -56,7 +56,6 @@ - com.github.jsqlparser jsqlparser diff --git a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java index 35c01fc5..c6e59090 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/app/OssieSalesforceConverter.java @@ -22,10 +22,8 @@ import org.apache.ossie.converter.Converter; import org.apache.ossie.converter.ConverterFactory; import org.apache.ossie.converter.ConversionDirection; -import org.apache.ossie.converter.SalesforceBindings; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.exception.InvalidInputException; -import org.apache.ossie.exception.ValidationException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -41,7 +39,6 @@ public class OssieSalesforceConverter { public static void main(String[] args) { if (args.length < 2) { - System.err.println("Usage: toSF [--bindings ] | toOssie "); System.exit(1); } @@ -51,26 +48,17 @@ public static void main(String[] args) { Path inputPath = Paths.get(args[1]); ConversionDirection direction = parseDirection(directionArg); - SalesforceBindings bindings = SalesforceBindings.none(); - if (args.length != 2) { - if (args.length != 4 || !"--bindings".equals(args[2]) - || direction != ConversionDirection.OSSIE_TO_SALESFORCE) { - throw new InvalidInputException("Expected toSF [--bindings ]"); - } - bindings = SalesforceBindings.fromPath(Paths.get(args[3])); - } - app.convert(direction, inputPath, bindings); + app.convert(direction, inputPath); } catch (InvalidInputException e) { - System.err.println("Error: " + e.getMessage()); System.exit(2); - } catch (ConversionException | ValidationException e) { + } catch (ConversionException e) { System.err.println("Error: " + e.getMessage()); System.exit(3); } } private static ConversionDirection parseDirection(String direction) { - return switch (direction.toLowerCase(java.util.Locale.ROOT)) { + return switch (direction.toLowerCase()) { case "tosf" -> ConversionDirection.OSSIE_TO_SALESFORCE; case "toossie" -> ConversionDirection.SALESFORCE_TO_OSSIE; default -> throw new InvalidInputException( @@ -87,10 +75,6 @@ private static ConversionDirection parseDirection(String direction) { * @param inputPath path to the input file */ public void convert(ConversionDirection direction, Path inputPath) { - convert(direction, inputPath, SalesforceBindings.none()); - } - - public void convert(ConversionDirection direction, Path inputPath, SalesforceBindings bindings) { if (!Files.exists(inputPath)) { throw new InvalidInputException("Input file not found: " + inputPath); } @@ -100,7 +84,7 @@ public void convert(ConversionDirection direction, Path inputPath, SalesforceBin outputDir = Path.of("."); } - Converter converter = ConverterFactory.getConverter(direction, bindings); + Converter converter = ConverterFactory.getConverter(direction); converter.convert(inputPath, outputDir); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java index 77f51b6f..88a51804 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java @@ -20,11 +20,9 @@ package org.apache.ossie.converter; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import org.apache.ossie.exception.ConversionException; @@ -80,18 +78,14 @@ protected AbstractConverter(PropertyMapper mapper) { this.mapper = mapper; this.jsonMapper = new ObjectMapper() - .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) - .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) .enable(SerializationFeature.INDENT_OUTPUT); YAMLFactory yamlFactory = new YAMLFactory() .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE); - yamlFactory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); this.yamlMapper = new ObjectMapper(yamlFactory) - .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) .enable(SerializationFeature.INDENT_OUTPUT); this.customExtensionHandler = new CustomExtensionHandler(this.jsonMapper); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java deleted file mode 100644 index bc9859ed..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionContext.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import java.util.Map; - -/** State owned by one model conversion; never shared across inputs or converter calls. */ -public final class ConversionContext { - private final Map sourceData; - private final Map outputData; - private FieldExpressionPlan fieldPlan; - - public ConversionContext(Map sourceData, Map outputData) { - this.sourceData = sourceData; - this.outputData = outputData; - } - - public Map sourceData() { return sourceData; } - public Map outputData() { return outputData; } - FieldExpressionPlan fieldPlan() { return fieldPlan; } - void fieldPlan(FieldExpressionPlan plan) { this.fieldPlan = plan; } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java index 2a7b0c4e..657e9504 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java @@ -34,9 +34,4 @@ public class ConverterFactory { public static Converter getConverter(ConversionDirection direction) { return new ConverterImpl(direction); } - - /** Creates a converter with an external environment binding catalog. */ - public static Converter getConverter(ConversionDirection direction, SalesforceBindings bindings) { - return new ConverterImpl(direction, bindings); - } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java index f942f5f7..c3735a4d 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ossie.converter.pipeline.*; +import org.apache.ossie.converter.pipeline.*; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; @@ -44,28 +45,14 @@ public class ConverterImpl extends AbstractConverter { private final DirectionConfig directionConfig; private final List steps; private final SchemaValidator schemaValidator; - private final SchemaValidator targetSchemaValidator; - private final SalesforceBindings bindings; public ConverterImpl(ConversionDirection direction) { - this(direction, PipelineConfigLoader.loadFromResource(), SalesforceBindings.none()); - } - - public ConverterImpl(ConversionDirection direction, SalesforceBindings bindings) { - this(direction, PipelineConfigLoader.loadFromResource(), bindings); + this(direction, PipelineConfigLoader.loadFromResource()); } ConverterImpl(ConversionDirection direction, PipelineConfig config) { - this(direction, config, SalesforceBindings.none()); - } - - private ConverterImpl(ConversionDirection direction, PipelineConfig config, SalesforceBindings bindings) { super(); this.direction = direction; - this.bindings = java.util.Objects.requireNonNull(bindings, "bindings"); - if (direction != ConversionDirection.OSSIE_TO_SALESFORCE && !bindings.isEmpty()) { - throw new ConversionException("Salesforce bindings apply only to toSF conversion"); - } // Get handler list for this direction List handlerNames = config.getPipelines().get(direction.toPipelineKey()); @@ -87,11 +74,6 @@ private ConverterImpl(ConversionDirection direction, PipelineConfig config, Sale directionConfig.getSchemaPath() ); - // Output validation is part of conversion, not only an optional test assertion. - this.targetSchemaValidator = new SchemaValidator(jsonMapper, - direction == ConversionDirection.OSSIE_TO_SALESFORCE - ? SchemaValidator.SALESFORCE_SCHEMA_PATH : SchemaValidator.OSSIE_SCHEMA_PATH); - // Initialize pipeline steps using factory HandlerFactory factory = new HandlerFactory(customExtensionHandler); this.steps = handlerNames.stream() @@ -117,12 +99,6 @@ public List convert(String content) { private List convertOssieToSalesforce(Map ossieRoot) { List semanticModels = getList(ossieRoot, SEMANTIC_MODEL); List results = new ArrayList<>(); - java.util.Set names = new java.util.HashSet<>(); - for (Object modelObj : semanticModels) { - String name = getString(asMap(modelObj), NAME); - if (!names.add(name)) throw new ConversionException("Duplicate model name '" + name + "'"); - } - bindings.validateModels(names); for (Object modelObj : semanticModels) { Map sourceData = asMap(modelObj); @@ -141,7 +117,6 @@ private List convertSalesforceToOssie(Map sourceData) { Map ossieRoot = new LinkedHashMap<>(); ossieRoot.put(VERSION, OSSIE_VERSION); ossieRoot.put(SEMANTIC_MODEL, List.of(outputData)); - targetSchemaValidator.validate(ossieRoot); return List.of(toYaml(ossieRoot)); } catch (JsonProcessingException e) { throw new ConversionException("Failed to wrap output in Ossie root", e); @@ -154,21 +129,10 @@ private String executePipeline(Map sourceData) { ? mapper.getOssieToSalesforceMappings() : mapper.getSalesforceToOssieMappings()); - ConversionContext context = new ConversionContext(sourceData, outputData); for (PipelineStep step : steps) { - try { - step.execute(context, mappings); - } catch (IllegalArgumentException e) { - String name = getString(sourceData, - direction == ConversionDirection.OSSIE_TO_SALESFORCE ? NAME : API_NAME); - throw new ConversionException("Model '" + name + "': " + e.getMessage(), e); - } - } - if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { - bindings.apply(sourceData, outputData); - new SalesforceModelValidator().validate(sourceData, outputData, context.fieldPlan()); - targetSchemaValidator.validate(outputData); + step.execute(sourceData, outputData, mappings); } + return serialize(outputData); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java index 81331838..849f6392 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/CustomExtensionHandler.java @@ -22,13 +22,9 @@ import static org.apache.ossie.converter.ConverterConstants.*; import static org.apache.ossie.util.DataStructureUtils.*; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ossie.converter.ConverterConstants.Level; -import org.apache.ossie.exception.ConversionException; import org.apache.ossie.util.PathUtils; import java.util.*; import org.slf4j.Logger; @@ -323,36 +319,31 @@ public void restoreSalesforceCustomExtension(Map sfItem, Map" : itemName) + "'"; - if (!(dataObj instanceof String dataJson)) { - throw new ConversionException(scope + " must contain a JSON object encoded as a string"); + if (dataObj == null) { + return; } - Map salesforceProperties; try { - // Use a strict reader without changing the shared mapper or the reverse conversion path. - salesforceProperties = jsonMapper.readerFor(new TypeReference>() {}) - .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) - .with(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) - .readValue(dataJson); - } catch (JsonProcessingException e) { - throw new ConversionException(scope + " must contain one JSON object with unique property names: " - + e.getOriginalMessage(), e); - } - if (salesforceProperties == null) { - throw new ConversionException(scope + " must contain a JSON object"); - } - - if (itemName == null) { - logger.warn("Item has no name, skipping custom_extensions restoration"); - return; - } + // Parse JSON string to Map + Map salesforceProperties = jsonMapper.readValue( + (String) dataObj, + new TypeReference>() {} + ); + + String itemName = getString(ossieItem, NAME); + if (itemName == null) { + logger.warn("Item has no name, skipping custom_extensions restoration"); + return; + } - for (Map.Entry entry : salesforceProperties.entrySet()) { - if (!sfItem.containsKey(entry.getKey())) { - sfItem.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + for (Map.Entry entry : salesforceProperties.entrySet()) { + if (!sfItem.containsKey(entry.getKey())) { + sfItem.put(entry.getKey(), PathUtils.deepCopyValue(entry.getValue())); + } } + + } catch (Exception e) { + logger.warn("Failed to restore custom_extensions: {}", e.getMessage()); } }); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java deleted file mode 100644 index 2e4d274c..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAnalyzer.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.converter.ExpressionAst.*; -import static org.apache.ossie.converter.ExpressionCompiler.Level; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** Resolves names and checks types, aggregation level and function domains before target rendering. */ -final class ExpressionAnalyzer { - private final String dialect; - private final ExpressionCompiler.ReferenceResolver resolver; - private int depth; - ExpressionAnalyzer(String dialect, ExpressionCompiler.ReferenceResolver resolver) { - this.dialect = dialect; this.resolver = resolver; - } - Typed analyze(Node node) { - if (++depth > 128) throw new IllegalArgumentException("expression nesting exceeds 128 levels"); - try { return analyzeNode(node); } finally { depth--; } - } - private Typed analyzeNode(Node node) { - if (node instanceof Literal literal) { - Object value = literal.value(); - Type type = value == null ? Type.NULL : value instanceof Boolean ? Type.BOOLEAN - : value instanceof String ? Type.STRING - : ((BigDecimal) value).stripTrailingZeros().scale() <= 0 ? Type.INTEGER : Type.DECIMAL; - return new Typed(node, type, Level.CONSTANT, Set.of(), List.of(), null, - value instanceof BigDecimal number ? number : null); - } - if (node instanceof Field field) { - ExpressionCompiler.Binding binding = resolver.resolve(field.reference()); - if (binding == null || binding.expression() == null || binding.expression().isBlank()) { - throw new IllegalArgumentException("field resolver returned no binding"); - } - Type type = Type.of(binding.datatype()); - if (type == Type.UNKNOWN) throw new IllegalArgumentException("field reference needs known field datatypes"); - return new Typed(node, type, binding.level(), binding.datasets(), List.of(), binding, null); - } - if (node instanceof Unary unary) { - Typed child = analyze(unary.operand()); - String operator = unary.operator(); - Type result = child.type(); - BigDecimal number = child.number(); - if (operator.equals("ISNULL")) { result = Type.BOOLEAN; number = null; } - else if (operator.equals("NOT")) { require(child, Type.BOOLEAN, "NOT"); result = Type.BOOLEAN; number = null; } - else { numeric(child, "unary " + operator); if (operator.equals("-") && number != null) number = number.negate(); } - return new Typed(node, result, child.level(), child.datasets(), List.of(child), null, number); - } - if (node instanceof Binary binary) { - Typed left = analyze(binary.left()); Typed right = analyze(binary.right()); - String op = binary.operator(); - Type result; - if (op.equals("AND") || op.equals("OR")) { - require(left, Type.BOOLEAN, op); require(right, Type.BOOLEAN, op); result = Type.BOOLEAN; - } else if (Set.of("=", "!=", "<", "<=", ">", ">=").contains(op)) { - compatible(left.type(), right.type(), "comparison"); - if (!Set.of("=", "!=").contains(op) && (left.type() == Type.BOOLEAN || right.type() == Type.BOOLEAN)) { - throw new IllegalArgumentException("ordered comparison requires numeric, text or temporal operands"); - } - result = Type.BOOLEAN; - } else { - numeric(left, op); numeric(right, op); - result = compatible(left.type(), right.type(), op); - if (op.equals("/")) { - if (right.number() != null && right.number().signum() == 0) { - throw new IllegalArgumentException("division by literal zero; use NULLIF(denominator, 0) for a nullable denominator"); - } - result = Type.DECIMAL; - } - } - return compose(node, result, List.of(left, right)); - } - if (node instanceof Conditional conditional) { - List children = new ArrayList<>(); - Type result = Type.NULL; - for (int i = 0; i < conditional.branches().size(); i += 2) { - Typed predicate = analyze(conditional.branches().get(i)); - require(predicate, Type.BOOLEAN, "conditional predicate"); - Typed branch = analyze(conditional.branches().get(i + 1)); - result = compatible(result, branch.type(), "conditional branches"); - children.add(predicate); children.add(branch); - } - Typed otherwise = analyze(conditional.otherwise()); - result = compatible(result, otherwise.type(), "conditional branches"); - children.add(otherwise); - return compose(node, result, children); - } - Call call = (Call) node; - ExpressionFunctionRegistry.Spec spec = ExpressionFunctionRegistry.require( - call.name(), dialect, call.arguments().size(), call.distinct()); - List arguments = call.arguments().stream().map(this::analyze).toList(); - return switch (spec.rule()) { - case AGGREGATE -> aggregate(call, arguments); - case COALESCE -> { - Type result = Type.NULL; - for (Typed argument : arguments) result = compatible(result, argument.type(), call.name() + " arguments"); - yield compose(node, result, arguments); - } - case NULLIF -> { - compatible(arguments.get(0).type(), arguments.get(1).type(), "NULLIF arguments"); - yield compose(node, arguments.get(0).type(), arguments); - } - case ISNULL -> compose(node, Type.BOOLEAN, arguments); - case NUMERIC -> numericFunction(call, arguments); - case YEAR -> { - Type input = arguments.get(0).type(); - if (!Set.of(Type.DATE, Type.DATETIME, Type.NULL).contains(input)) { - throw new IllegalArgumentException("YEAR requires Date or DateTime; timezone-dependent extraction is unsupported"); - } - yield compose(node, Type.INTEGER, arguments); - } - case LENGTH -> { - require(arguments.get(0), Type.STRING, call.name()); - yield compose(node, Type.INTEGER, arguments); - } - case POSITION -> { - for (Typed argument : arguments) require(argument, Type.STRING, call.name()); - yield compose(node, Type.INTEGER, arguments); - } - case SUBSTRING -> substring(call, arguments); - }; - } - private Typed aggregate(Call call, List arguments) { - String name = call.name(); Typed argument = arguments.get(0); - if (argument.level() == Level.AGGREGATE) throw new IllegalArgumentException("nested aggregate " + name + " is unsupported"); - if (argument.datasets().isEmpty()) throw new IllegalArgumentException(name + " needs a declared field to establish its dataset"); - if (argument.datasets().size() > 1) throw new IllegalArgumentException("one aggregate cannot combine fields from multiple datasets"); - boolean count = name.equals("COUNT") || name.equals("COUNTD"); - if (count && !(argument.node() instanceof Field)) { - throw new IllegalArgumentException(name + " requires a declared field; counting expressions is unsupported"); - } - if (name.equals("MIN") || name.equals("MAX")) { - if (argument.type() == Type.BOOLEAN || argument.type() == Type.UNKNOWN) { - throw new IllegalArgumentException(name + " requires numeric, text or temporal operands"); - } - } else if (!count) numeric(argument, name); - Type result = count ? Type.INTEGER : name.equals("AVG") ? Type.DECIMAL : argument.type(); - return new Typed(call, result, Level.AGGREGATE, argument.datasets(), arguments, null, null); - } - private Typed numericFunction(Call call, List arguments) { - Typed value = arguments.get(0); numeric(value, call.name()); - BigDecimal places = BigDecimal.ZERO; - if (arguments.size() == 2) { - places = arguments.get(1).number(); - if (places == null || places.stripTrailingZeros().scale() > 0 - || places.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0 - || places.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { - throw new IllegalArgumentException("ROUND precision must be a 32-bit integer literal"); - } - } - Type result = value.type(); - if (result != Type.NULL && (Set.of("CEIL", "CEILING", "FLOOR").contains(call.name()) - || call.name().equals("ROUND") && places.signum() <= 0)) result = Type.INTEGER; - return compose(call, result, arguments); - } - private Typed substring(Call call, List arguments) { - require(arguments.get(0), Type.STRING, call.name()); - for (int i = 1; i < arguments.size(); i++) require(arguments.get(i), Type.INTEGER, call.name()); - // Snowflake allows non-positive indices with semantics different from Tua MID. - // Only proven common domains are lowered, including POSITION(...) + 1 in email-domain fields. - if (call.name().equals("SUBSTRING")) { - BigDecimal start = lowerBound(arguments.get(1)); - if (start == null || start.signum() <= 0) throw new IllegalArgumentException("SUBSTRING start must be provably positive for Tua MID"); - if (arguments.size() == 3) { - BigDecimal length = lowerBound(arguments.get(2)); - if (length == null || length.signum() < 0) throw new IllegalArgumentException("SUBSTRING length must be provably non-negative for Tua MID"); - } - } - return compose(call, Type.STRING, arguments); - } - private BigDecimal lowerBound(Typed value) { - if (value.number() != null) return value.number(); - if (value.node() instanceof Call call && Set.of("LENGTH", "LEN", "POSITION", "FIND").contains(call.name())) return BigDecimal.ZERO; - if (value.node() instanceof Binary binary && binary.operator().equals("+")) { - BigDecimal left = lowerBound(value.children().get(0)), right = lowerBound(value.children().get(1)); - return left == null || right == null ? null : left.add(right); - } - return null; - } - private Typed compose(Node node, Type type, List arguments) { - Level level = Level.CONSTANT; Set datasets = new HashSet<>(); - for (Typed argument : arguments) { - if (level != Level.CONSTANT && argument.level() != Level.CONSTANT && level != argument.level()) { - throw new IllegalArgumentException("cannot mix aggregate and unaggregated field expressions"); - } - if (argument.level() != Level.CONSTANT) level = argument.level(); - datasets.addAll(argument.datasets()); - } - return new Typed(node, type, level, datasets, arguments, null, null); - } - private static void numeric(Typed value, String context) { - if (!value.type().numeric() && value.type() != Type.NULL) { - throw new IllegalArgumentException(context + " requires numeric operands, found " + value.type() + "; declare a compatible field datatype"); - } - } - private static void require(Typed value, Type expected, String context) { - if (value.type() != expected && value.type() != Type.NULL) throw new IllegalArgumentException(context + " requires " + expected + ", found " + value.type()); - } - private static Type compatible(Type left, Type right, String context) { - if (left == Type.UNKNOWN || right == Type.UNKNOWN) throw new IllegalArgumentException(context + " needs known field datatypes"); - if (left == Type.NULL) return right; - if (right == Type.NULL || left == right) return left; - if (left.numeric() && right.numeric()) return left == Type.FLOAT || right == Type.FLOAT ? Type.FLOAT : Type.DECIMAL; - throw new IllegalArgumentException(context + " has incompatible types " + left + " and " + right); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java deleted file mode 100644 index 45af5c4b..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionCompiler.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.util.DataStructureUtils.*; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** Shared expression compiler for derived fields and metrics. Target emission follows semantic checks. */ -final class ExpressionCompiler { - private static final List DIALECTS = List.of("TABLEAU", "SNOWFLAKE", "ANSI_SQL"); - private ExpressionCompiler() {} - - enum Level { CONSTANT, ROW, AGGREGATE } - record Selected(String text, String dialect) {} - record Parsed(ExpressionAst.Node root, String dialect) {} - record Reference(List parts, boolean tableau) { - Reference { parts = List.copyOf(parts); } - } - record Binding(String expression, String datatype, Set datasets, Level level) { - Binding { datasets = Set.copyOf(datasets); } - Binding(String expression, String datatype, String dataset, Level level) { - this(expression, datatype, dataset == null ? Set.of() : Set.of(dataset), level); - } - Binding(String expression, String datatype, String dataset) { - this(expression, datatype, dataset, Level.ROW); - } - String dataset() { return datasets.size() == 1 ? datasets.iterator().next() : null; } - } - record Compiled(String expression, String datatype, Level level, Set datasets) { - Compiled { datasets = Set.copyOf(datasets); } - } - @FunctionalInterface interface ReferenceResolver { Binding resolve(Reference reference); } - - static Selected select(Map owner) { - Map expression = getMap(owner, "expression"); - List dialects = expression == null ? null : getList(expression, "dialects"); - if (dialects == null) { - throw new IllegalArgumentException("missing expression.dialects; provide TABLEAU, SNOWFLAKE or ANSI_SQL"); - } - Map candidates = new LinkedHashMap<>(); - for (Object entry : dialects) { - Map value = asMap(entry); - String dialect = getString(value, "dialect"); - if (DIALECTS.contains(dialect)) { - if (candidates.containsKey(dialect)) { - throw new IllegalArgumentException("ambiguous expression: multiple " + dialect + " entries"); - } - candidates.put(dialect, getString(value, "expression")); - } - } - String dialect = DIALECTS.stream().filter(candidates::containsKey).findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "no supported dialect; provide TABLEAU, SNOWFLAKE or ANSI_SQL")); - String text = candidates.get(dialect); - if (text == null || text.isBlank()) throw new IllegalArgumentException(dialect + " expression is empty"); - return new Selected(text, dialect); - } - - static Parsed parse(String text, String dialect) { - if (!DIALECTS.contains(dialect)) throw new IllegalArgumentException("unsupported expression dialect " + dialect); - if (text == null || text.isBlank()) throw new IllegalArgumentException(dialect + " expression is empty"); - List tokens = ExpressionTokens.tokenize(text, dialect); - ExpressionAst.Node root = dialect.equals("TABLEAU") - ? new TuaExpressionParser(tokens).parse() : SqlExpressionParser.parse(text, dialect); - return new Parsed(root, dialect); - } - - static Optional directReference(Parsed parsed) { - return parsed.root() instanceof ExpressionAst.Field field ? Optional.of(field.reference()) : Optional.empty(); - } - - static Compiled compile(Parsed parsed, ReferenceResolver resolver) { - ExpressionAst.Typed checked = new ExpressionAnalyzer(parsed.dialect(), resolver).analyze(parsed.root()); - String expression = new TuaExpressionEmitter().emit(checked); - return new Compiled(expression, checked.type().datatype, checked.level(), checked.datasets()); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java deleted file mode 100644 index b5a585b2..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionFunctionRegistry.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import java.util.Map; -import java.util.Set; - -/** Closed, version-controlled target capabilities; parser acceptance never implies function support. */ -final class ExpressionFunctionRegistry { - enum Rule { AGGREGATE, COALESCE, NULLIF, ISNULL, NUMERIC, YEAR, LENGTH, POSITION, SUBSTRING } - record Spec(String target, int min, int max, Rule rule, Set dialects) {} - private static final Set ALL = Set.of("SNOWFLAKE", "ANSI_SQL", "TABLEAU"); - private static final Set SQL = Set.of("SNOWFLAKE", "ANSI_SQL"); - private static final Set TUA = Set.of("TABLEAU"); - private static Spec spec(String target, int min, int max, Rule rule, Set dialects) { - return new Spec(target, min, max, rule, dialects); - } - private static final Map FUNCTIONS = Map.ofEntries( - Map.entry("SUM", spec("SUM", 1, 1, Rule.AGGREGATE, ALL)), - Map.entry("AVG", spec("AVG", 1, 1, Rule.AGGREGATE, ALL)), - Map.entry("MIN", spec("MIN", 1, 1, Rule.AGGREGATE, ALL)), - Map.entry("MAX", spec("MAX", 1, 1, Rule.AGGREGATE, ALL)), - Map.entry("COUNT", spec("COUNT", 1, 1, Rule.AGGREGATE, ALL)), - Map.entry("COUNTD", spec("COUNTD", 1, 1, Rule.AGGREGATE, TUA)), - Map.entry("COALESCE", spec("IFNULL", 2, Integer.MAX_VALUE, Rule.COALESCE, SQL)), - Map.entry("IFNULL", spec("IFNULL", 2, 2, Rule.COALESCE, TUA)), - Map.entry("NULLIF", spec("IF", 2, 2, Rule.NULLIF, SQL)), - Map.entry("ISNULL", spec("ISNULL", 1, 1, Rule.ISNULL, TUA)), - Map.entry("ABS", spec("ABS", 1, 1, Rule.NUMERIC, ALL)), - Map.entry("CEIL", spec("CEILING", 1, 1, Rule.NUMERIC, SQL)), - Map.entry("CEILING", spec("CEILING", 1, 1, Rule.NUMERIC, TUA)), - Map.entry("FLOOR", spec("FLOOR", 1, 1, Rule.NUMERIC, ALL)), - Map.entry("ROUND", spec("ROUND", 1, 2, Rule.NUMERIC, ALL)), - Map.entry("YEAR", spec("YEAR", 1, 1, Rule.YEAR, ALL)), - Map.entry("LENGTH", spec("LEN", 1, 1, Rule.LENGTH, SQL)), - Map.entry("LEN", spec("LEN", 1, 1, Rule.LENGTH, TUA)), - Map.entry("POSITION", spec("FIND", 2, 2, Rule.POSITION, SQL)), - Map.entry("FIND", spec("FIND", 2, 2, Rule.POSITION, TUA)), - Map.entry("SUBSTRING", spec("MID", 2, 3, Rule.SUBSTRING, SQL)), - Map.entry("MID", spec("MID", 2, 3, Rule.SUBSTRING, TUA))); - - private ExpressionFunctionRegistry() {} - static Spec require(String name, String dialect, int count, boolean distinct) { - Spec spec = FUNCTIONS.get(name); - if (spec == null) throw new IllegalArgumentException("unsupported function " + name); - if (!spec.dialects().contains(dialect)) { - throw new IllegalArgumentException(name + " is outside the supported " + dialect + " subset"); - } - if (distinct && (!name.equals("COUNT") || dialect.equals("TABLEAU"))) { - throw new IllegalArgumentException("DISTINCT is supported only by SQL COUNT(DISTINCT field)"); - } - if (count < spec.min() || count > spec.max()) { - throw new IllegalArgumentException(name + " expects " - + (spec.min() == spec.max() ? spec.min() : spec.min() + " to " + spec.max()) + " arguments"); - } - return spec; - } - static Spec get(String name) { return FUNCTIONS.get(name); } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java deleted file mode 100644 index a39c1998..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionTokens.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -/** Bounded lexical preflight shared by the SQL and native Tua frontends. */ -final class ExpressionTokens { - enum Kind { WORD, IDENTIFIER, STRING, NUMBER, SYMBOL, END } - record Token(Kind kind, String text, int offset, boolean bracket) {} - private ExpressionTokens() {} - static List tokenize(String text, String dialect) { - if (text.length() > 32768) throw new IllegalArgumentException("expression exceeds 32768 characters"); - List tokens = new ArrayList<>(); - int nesting = 0; - for (int i = 0; i < text.length();) { - char c = text.charAt(i); - if (Character.isWhitespace(c)) { i++; continue; } - int start = i; - if (c == '\'' || c == '"' || c == '[') { - if (c == '[' && dialect.equals("SNOWFLAKE")) throw lexical(dialect, i, "use double-quoted SQL identifiers"); - boolean string = c == '\'' || (c == '"' && dialect.equals("TABLEAU")); - char end = c == '[' ? ']' : c; - StringBuilder value = new StringBuilder(); - boolean closed = false; - i++; - while (i < text.length()) { - char part = text.charAt(i++); - if (part == end) { - if (i < text.length() && text.charAt(i) == end) { value.append(end); i++; } - else { closed = true; break; } - } else { - if (Character.isISOControl(part) || (string && part == '\\')) { - throw lexical(dialect, i - 1, "control characters and backslash string escapes are unsupported"); - } - value.append(part); - } - } - if (!closed) throw lexical(dialect, start, "unterminated quoted value"); - tokens.add(new Token(string ? Kind.STRING : Kind.IDENTIFIER, value.toString(), start, c == '[')); - } else if (Character.isDigit(c) || (c == '.' && i + 1 < text.length() && Character.isDigit(text.charAt(i + 1)))) { - i++; - while (i < text.length() && (Character.isDigit(text.charAt(i)) || text.charAt(i) == '.')) i++; - if (i < text.length() && (text.charAt(i) == 'e' || text.charAt(i) == 'E')) { - i++; - if (i < text.length() && (text.charAt(i) == '+' || text.charAt(i) == '-')) i++; - while (i < text.length() && Character.isDigit(text.charAt(i))) i++; - } - tokens.add(new Token(Kind.NUMBER, text.substring(start, i), start, false)); - } else if (Character.isLetter(c) || c == '_') { - i++; - while (i < text.length() && (Character.isLetterOrDigit(text.charAt(i)) || text.charAt(i) == '_' || text.charAt(i) == '$')) i++; - tokens.add(new Token(Kind.WORD, text.substring(start, i), start, false)); - } else { - if (i + 1 < text.length() && (text.startsWith("--", i) || text.startsWith("/*", i))) { - throw lexical(dialect, i, "comments are unsupported in metric expressions"); - } - String symbol = String.valueOf(c); - if (i + 1 < text.length() && Set.of("<=", ">=", "<>", "!=").contains(text.substring(i, i + 2))) { - symbol = text.substring(i, i + 2); - i++; - } - if (!"()+-*/.,=<>!".contains(String.valueOf(c))) throw lexical(dialect, start, "unsupported character '" + c + "'"); - tokens.add(new Token(Kind.SYMBOL, symbol, start, false)); - i++; - } - Token added = tokens.get(tokens.size() - 1); - if (added.kind() == Kind.NUMBER) number(added.text()); - if (added.kind() == Kind.SYMBOL && added.text().equals("(") && ++nesting > 128) { - throw lexical(dialect, start, "expression nesting exceeds 128 levels"); - } - if (added.kind() == Kind.SYMBOL && added.text().equals(")")) nesting--; - if (tokens.size() > 8192) throw lexical(dialect, start, "too many expression tokens"); - } - if (nesting > 0) throw lexical(dialect, text.length(), "expected )"); - tokens.add(new Token(Kind.END, "end of expression", text.length(), false)); - return tokens; - } - - private static IllegalArgumentException lexical(String dialect, int offset, String message) { - return new IllegalArgumentException(dialect + " at character " + (offset + 1) + ": " + message); - } - static BigDecimal number(String text) { - BigDecimal value; - try { value = new BigDecimal(text); } - catch (NumberFormatException e) { throw new IllegalArgumentException("invalid numeric literal '" + text + "'"); } - if (Math.abs((long) value.scale()) > 1000 || value.precision() > 1000) { - throw new IllegalArgumentException("numeric literal is too large"); - } - return value; - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java deleted file mode 100644 index 071a94d7..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldExpressionPlan.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.util.DataStructureUtils.getList; -import static org.apache.ossie.util.DataStructureUtils.getString; -import static org.apache.ossie.util.DataStructureUtils.streamMaps; - -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.HexFormat; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import org.apache.ossie.converter.ExpressionCompiler.Binding; -import org.apache.ossie.converter.ExpressionCompiler.Compiled; -import org.apache.ossie.converter.ExpressionCompiler.Level; -import org.apache.ossie.converter.ExpressionCompiler.Parsed; -import org.apache.ossie.converter.ExpressionCompiler.Reference; -import org.apache.ossie.converter.ExpressionCompiler.Selected; -import org.apache.ossie.converter.MetricFieldResolver.Identifier; - -/** - * A per-conversion field dependency plan. SQL field expressions have dataset-local physical - * column scope plus explicitly declared semantic aliases; metric scope remains semantic only. - * Calculated aliases are expanded over direct semantic fields, never guessed global Tua names. - */ -final class FieldExpressionPlan { - record PlannedField(String dataset, String name, Map source, - Parsed parsed, Reference directReference, String physicalColumn, String calculatedApiName) { - boolean direct() { return physicalColumn != null; } - } - - private record Key(String dataset, String field) { - @Override public String toString() { return dataset + "." + field; } - } - - private final Map> datasets = new LinkedHashMap<>(); - private final Map fields = new LinkedHashMap<>(); - private final Map> byDataset = new LinkedHashMap<>(); - private final Map resolved = new HashMap<>(); - private final Map> lineage = new HashMap<>(); - private final Map>> sqlReferences = new HashMap<>(); - private final Map>> tableauReferences = new HashMap<>(); - private final Map>> qualifiers = new HashMap<>(); - private final Map> exported = new HashMap<>(); - private final Map target; - private final ArrayDeque pending = new ArrayDeque<>(); - private boolean indexed; - - FieldExpressionPlan(Map source, Map target) { - this.target = target; - Set reserved = new HashSet<>(); - for (String list : List.of("metrics", "semanticCalculatedDimensions", "semanticCalculatedMeasurements")) { - for (Map item : items(list.equals("metrics") ? source : target, list)) { - String name = getString(item, list.equals("metrics") ? "name" : "apiName"); - if (name != null) reserved.add(name.toUpperCase(Locale.ROOT)); - } - } - for (Map dataset : items(source, "datasets")) { - String datasetName = requiredName(dataset, "dataset"); - if (datasets.putIfAbsent(datasetName, dataset) != null) { - throw new IllegalArgumentException("Duplicate dataset declaration '" + datasetName + "'"); - } - byDataset.put(datasetName, new ArrayList<>()); - qualifiers.put(datasetName, sourceQualifiers(dataset)); - for (Map field : items(dataset, "fields")) { - String fieldName = requiredName(field, "field in dataset '" + datasetName + "'"); - Key key = new Key(datasetName, fieldName); - try { - Selected selected = ExpressionCompiler.select(field); - Parsed parsed = ExpressionCompiler.parse(selected.text(), selected.dialect()); - Reference reference = ExpressionCompiler.directReference(parsed).orElse(null); - // A Tua reference addresses semantic fields, not physical catalog columns. - String physical = reference != null && !reference.tableau() - ? physicalColumn(datasetName, reference) : null; - PlannedField planned = new PlannedField(datasetName, fieldName, field, parsed, - physical == null ? null : reference, physical, null); - if (fields.putIfAbsent(key, planned) != null) { - throw new IllegalArgumentException("Duplicate field declaration"); - } - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Field '" + key + "': " + e.getMessage(), e); - } - } - } - // Reserve every ordinary name before assigning suffixes, independent of declaration order. - Map candidateCounts = new HashMap<>(); - for (PlannedField field : fields.values()) { - if (!field.direct()) candidateCounts.merge(candidate(field).toUpperCase(Locale.ROOT), 1, Integer::sum); - } - for (Key key : fields.keySet().stream().sorted(Comparator.comparing(Key::dataset).thenComparing(Key::field)).toList()) { - PlannedField field = fields.get(key); - if (!field.direct()) { - String base = candidate(field); - String name = base; - if (candidateCounts.get(base.toUpperCase(Locale.ROOT)) > 1 - || reserved.contains(base.toUpperCase(Locale.ROOT))) { - String hash = digest(key.dataset() + "\u0000" + key.field()); - int length = 12; - do { - if (length > hash.length()) { - throw new IllegalArgumentException("Cannot allocate unique calculated field name for '" + key + "'"); - } - name = base + "__" + hash.substring(0, length); - length += 4; - } while (reserved.contains(name.toUpperCase(Locale.ROOT)) - || candidateCounts.containsKey(name.toUpperCase(Locale.ROOT))); - } - reserved.add(name.toUpperCase(Locale.ROOT)); - fields.put(key, new PlannedField(field.dataset(), field.name(), field.source(), field.parsed(), - null, null, name)); - } - } - for (PlannedField field : fields.values()) byDataset.get(field.dataset()).add(field); - byDataset.replaceAll((key, value) -> List.copyOf(value)); - for (PlannedField field : fields.values()) { - addIndex(tableauReferences, field.dataset(), field.name(), field); - addIndex(sqlReferences, field.dataset(), normalizeDeclaration(field.name()), field); - if (field.direct()) { - List parts = field.directReference().parts(); - addIndex(sqlReferences, field.dataset(), normalize(parts.get(parts.size() - 1)), field); - } - } - } - - List fields(String dataset) { return byDataset.getOrDefault(dataset, List.of()); } - - boolean isDirect(String dataset, String field) { return field(dataset, field).direct(); } - - String calculatedApiName(String dataset, String field) { return field(dataset, field).calculatedApiName(); } - - boolean hasPhysicalDependencies(String dataset, String field) { - resolve(dataset, field); - return !lineage.get(new Key(dataset, field)).isEmpty(); - } - - List> dependencies(String dataset, String field) { - resolve(dataset, field); - return lineage.get(new Key(dataset, field)).stream() - .sorted(Comparator.comparing(Key::dataset).thenComparing(Key::field)) - .map(key -> Map.of("dependentDefinitionApiName", key.dataset(), - "dependentFieldApiName", key.field())).toList(); - } - - /** Compile all derived fields, including unused ones, so unsupported fields cannot disappear. */ - void compileAll() { - indexExported(); - for (PlannedField field : fields.values()) { - if (!field.direct()) resolve(field.dataset(), field.name()); - } - } - - /** Exact declaration lookup for a metric resolver that has already resolved identifier spelling. */ - Binding resolve(String dataset, String field) { - indexExported(); - Key key = new Key(dataset, field); - Binding cached = resolved.get(key); - if (cached != null) return cached; - PlannedField planned = field(dataset, field); - if (planned.direct()) { - Map output = exported.get(key); - if (output == null) throw new IllegalArgumentException("Field '" + key + "' was not exported as a direct field"); - String datatype = getString(planned.source(), "datatype"); - String targetType = getString(output, "dataType"); - if (datatype == null || datatype.isBlank()) datatype = SalesforceDataTypeMapper.toOssie(targetType); - if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { - throw new IllegalArgumentException("Field '" + key + "' has no supported datatype; declare a portable field datatype"); - } - if (targetType == null || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { - throw new IllegalArgumentException("Field '" + key + "' datatype conflicts with exported Salesforce dataType '" + targetType + "'"); - } - Binding binding = new Binding(bracket(dataset) + "." + bracket(field), datatype, dataset, Level.ROW); - resolved.put(key, binding); - lineage.put(key, Set.of(key)); - return binding; - } - if (pending.contains(key)) { - throw new IllegalArgumentException("Calculated field dependency cycle: " - + String.join(" -> ", pending.stream().map(Key::toString).toList()) + " -> " + key); - } - if (pending.size() >= 128) { - throw new IllegalArgumentException("Calculated field dependency depth exceeds 128 at '" + key + "'"); - } - pending.addLast(key); - try { - Set dependencies = new LinkedHashSet<>(); - Compiled compiled = ExpressionCompiler.compile(planned.parsed(), reference -> bind(planned, reference, dependencies)); - if (compiled.level() == Level.AGGREGATE) { - throw new IllegalArgumentException("Dataset fields must be row expressions; declare aggregate expressions as metrics"); - } - String declared = getString(planned.source(), "datatype"); - String datatype = compiled.datatype(); - if (declared != null) { - if (!compatibleResult(declared, datatype)) { - throw new IllegalArgumentException("Declared datatype '" + declared - + "' conflicts with calculated result datatype '" + datatype + "'"); - } - datatype = declared; - } - if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { - throw new IllegalArgumentException("Calculated field needs a supported, unambiguous datatype"); - } - // Constant fields still belong to their declared dataset when consumed by a metric. - Binding binding = new Binding(compiled.expression(), datatype, dataset, Level.ROW); - resolved.put(key, binding); - lineage.put(key, Set.copyOf(dependencies)); - return binding; - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Field '" + key + "': " + e.getMessage(), e); - } finally { - pending.removeLast(); - } - } - - private Binding bind(PlannedField owner, Reference reference, Set dependencies) { - List parts = reference.parts(); - String text = String.join(".", parts.stream().map(Identifier::text).toList()); - if (parts.isEmpty()) throw new IllegalArgumentException("Empty field reference"); - if (reference.tableau()) { - if (parts.size() != 2 || !owner.dataset().equals(parts.get(0).text())) { - throw new IllegalArgumentException("Row field reference '" + text - + "' must use [" + owner.dataset() + "].[declared field]; cross-dataset row calculations are unsupported"); - } - } else if (!validQualifier(owner.dataset(), parts.subList(0, parts.size() - 1))) { - throw new IllegalArgumentException("Unknown physical or semantic dataset qualifier in row field reference '" + text + "'"); - } - Identifier name = parts.get(parts.size() - 1); - Set matches = (reference.tableau() ? tableauReferences : sqlReferences) - .getOrDefault(owner.dataset(), Map.of()) - .getOrDefault(reference.tableau() ? name.text() : normalize(name), Set.of()); - if (matches.isEmpty()) { - throw new IllegalArgumentException("Unknown row field reference '" + text - + "'; declare its physical column or semantic field in dataset '" + owner.dataset() + "'"); - } - if (matches.size() > 1) { - throw new IllegalArgumentException("Ambiguous row field reference '" + text - + "' matches multiple declared physical columns or semantic fields"); - } - PlannedField match = matches.iterator().next(); - Binding binding = resolve(match.dataset(), match.name()); - dependencies.addAll(lineage.get(new Key(match.dataset(), match.name()))); - return binding; - } - - private void indexExported() { - if (indexed) return; - for (Map dataset : items(target, "semanticDataObjects")) { - String name = getString(dataset, "apiName"); - for (String kind : List.of("semanticDimensions", "semanticMeasurements")) { - for (Map item : items(dataset, kind)) { - Key key = new Key(name, getString(item, "apiName")); - if (exported.putIfAbsent(key, item) != null) { - throw new IllegalArgumentException("Duplicate exported field '" + key + "'"); - } - } - } - } - indexed = true; - } - - private PlannedField field(String dataset, String field) { - PlannedField planned = fields.get(new Key(dataset, field)); - if (planned == null) throw new IllegalArgumentException("Unknown declared field '" + dataset + "." + field + "'"); - return planned; - } - - private String physicalColumn(String dataset, Reference reference) { - List parts = reference.parts(); - if (parts.isEmpty() || !validQualifier(dataset, parts.subList(0, parts.size() - 1))) { - throw new IllegalArgumentException("Direct field qualifier does not match its declared dataset or physical source"); - } - return parts.get(parts.size() - 1).text(); - } - - private boolean validQualifier(String dataset, List qualifier) { - return qualifier.isEmpty() || qualifiers.get(dataset).contains(qualifier.stream() - .map(FieldExpressionPlan::normalize).toList()); - } - - private static Set> sourceQualifiers(Map dataset) { - Set> result = new HashSet<>(); - result.add(List.of(normalizeDeclaration(getString(dataset, "name")))); - String source = getString(dataset, "source"); - if (source != null) { - try { - Reference ref = ExpressionCompiler.directReference(ExpressionCompiler.parse(source, "ANSI_SQL")).orElse(null); - if (ref != null) { - List parts = ref.parts().stream().map(FieldExpressionPlan::normalize).toList(); - for (int i = 0; i < parts.size(); i++) result.add(List.copyOf(parts.subList(i, parts.size()))); - } - } catch (IllegalArgumentException ignored) { - // Opaque external source identifiers do not create implicit SQL aliases. - } - } - return Set.copyOf(result); - } - - private static void addIndex(Map>> index, - String dataset, String name, PlannedField field) { - index.computeIfAbsent(dataset, key -> new HashMap<>()) - .computeIfAbsent(name, key -> new LinkedHashSet<>()).add(field); - } - - private static boolean compatibleResult(String declared, String inferred) { - if (SalesforceDataTypeMapper.toSalesforce(declared) == null) return false; - if (inferred == null || declared.equals(inferred)) return true; - return Set.of("Decimal", "Float").contains(declared) - && Set.of("Integer", "Decimal", "Float").contains(inferred); - } - - private static String candidate(PlannedField field) { - String name = (field.dataset() + "__" + field.name()).replaceAll("[^A-Za-z0-9_]", "_"); - if (name.isEmpty() || Character.isDigit(name.charAt(0))) name = "field_" + name; - return name; - } - - private static String digest(String value) { - try { - return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("SHA-256 is required by the Java runtime", e); - } - } - - private static String bracket(String name) { - if (name == null || name.isBlank() || name.contains("[") || name.contains("]") - || name.chars().anyMatch(Character::isISOControl)) { - throw new IllegalArgumentException("Semantic name '" + name + "' cannot be represented safely in Tua"); - } - return "[" + name + "]"; - } - - private static String normalize(Identifier name) { - return name.quoted() ? name.text() : name.text().toUpperCase(Locale.ROOT); - } - - private static String normalizeDeclaration(String name) { - if (name.startsWith("\"") && name.endsWith("\"") && name.length() >= 2) { - String text = name.substring(1, name.length() - 1); - if (text.isEmpty() || text.replace("\"\"", "").contains("\"")) { - throw new IllegalArgumentException("Invalid quoted declaration name '" + name + "'"); - } - return text.replace("\"\"", "\""); - } - return name.toUpperCase(Locale.ROOT); - } - - private static String requiredName(Map item, String kind) { - String name = getString(item, "name"); - if (name == null || name.isBlank()) throw new IllegalArgumentException("Missing name for " + kind); - return name; - } - - private static List> items(Map map, String name) { - List list = getList(map, name); - return list == null ? List.of() : streamMaps(list).toList(); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java index f2d79763..4147e0ef 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java @@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.regex.Pattern; import static org.apache.ossie.converter.ConverterConstants.*; import static org.apache.ossie.util.DataStructureUtils.*; @@ -43,9 +44,27 @@ public class FieldMappingHandler implements PipelineStep { private static final Set SF_FIELD_HANDLED_PROPS = Set.of(API_NAME, LABEL, DESCRIPTION, DATA_OBJECT_FIELD_NAME); + // Compiled regex pattern for SQL keywords that indicate calculated expressions + private static final Pattern CALCULATED_KEYWORDS_PATTERN = Pattern.compile( + "\\b(CASE|WHEN|THEN|ELSE|END|CAST|CONVERT|EXTRACT|SUBSTRING|SUBSTR|" + + "COALESCE|NULLIF|IFNULL|CONCAT|UPPER|LOWER|TRIM|LENGTH|" + + "AND|OR|NOT|IN|BETWEEN|LIKE|IS\\s+NULL|IS\\s+NOT\\s+NULL|DISTINCT|" + + "COUNT|SUM|AVG|MIN|MAX|DATE|YEAR|MONTH|DAY)\\b" + ); + private final ConversionDirection direction; private final CustomExtensionHandler customExtensionHandler; + /** + * Enum representing the four possible field types in Salesforce Semantic Model. + */ + private enum FieldType { + DIMENSION, // Direct dimension: !isCalculated + hasDimension + MEASUREMENT, // Direct measurement: !isCalculated + !hasDimension + CALCULATED_DIMENSION, // Calculated dimension: isCalculated + hasDimension + CALCULATED_MEASUREMENT // Calculated measurement: isCalculated + !hasDimension + } + public FieldMappingHandler(ConversionDirection direction, CustomExtensionHandler customExtensionHandler) { this.direction = direction; this.customExtensionHandler = customExtensionHandler; @@ -56,72 +75,38 @@ public FieldMappingHandler(ConversionDirection direction, CustomExtensionHandler */ @Override public void execute(Map sourceData, Map outputData, Map mappings) { - execute(new ConversionContext(sourceData, outputData), mappings); - } - - @Override - public void execute(ConversionContext context, Map mappings) { logger.debug("Mapping fields in {} direction", direction); if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { - FieldExpressionPlan plan = new FieldExpressionPlan(context.sourceData(), context.outputData()); - mapOssieToSalesforce(context.sourceData(), context.outputData(), plan); - context.fieldPlan(plan); + mapOssieToSalesforce(sourceData, outputData); } else { - mapSalesforceToOssie(context.sourceData(), context.outputData()); + mapSalesforceToOssie(sourceData, outputData); } } - /** Emit physical bindings first, then compile every derived field against those bindings. */ - private void mapOssieToSalesforce(Map sourceData, - Map outputData, FieldExpressionPlan plan) { - List targets = getList(outputData, SEMANTIC_DATA_OBJECTS); - Map> targetsByName = new LinkedHashMap<>(); - if (targets != null) { - for (Object item : targets) { - Map target = asMap(item); - String name = getString(target, API_NAME); - if (targetsByName.putIfAbsent(name, target) != null) { - throw new IllegalArgumentException("Duplicate exported dataset '" + name + "'"); - } - } - } - for (Object item : getList(sourceData, DATASETS)) { - Map dataset = asMap(item); - String datasetName = getString(dataset, NAME); - Map target = targetsByName.get(datasetName); - if (target == null) { - throw new IllegalArgumentException("Dataset '" + datasetName + "' was not exported before field mapping"); - } - for (FieldExpressionPlan.PlannedField field : plan.fields(datasetName)) { - if (!field.direct()) continue; - Map sfField = mapFieldProperties(field.source(), field.physicalColumn()); - customExtensionHandler.restoreSalesforceCustomExtension(sfField, field.source()); - applyOssieDatatype(sfField, field.source()); - applyFieldDefaults(sfField); - getOrCreateList(target, field.source().containsKey(DIMENSION) - ? SEMANTIC_DIMENSIONS : SEMANTIC_MEASUREMENTS).add(sfField); - } - } - plan.compileAll(); - for (Object item : getList(sourceData, DATASETS)) { - String datasetName = getString(asMap(item), NAME); - for (FieldExpressionPlan.PlannedField field : plan.fields(datasetName)) { - if (field.direct()) continue; - ExpressionCompiler.Binding binding = plan.resolve(datasetName, field.name()); - Map calc = createSemanticCalculatedDimension(field.source(), binding.expression()); - calc.put(API_NAME, field.calculatedApiName()); - calc.put(DEPENDENCIES, plan.dependencies(datasetName, field.name())); - customExtensionHandler.restoreSalesforceCustomExtension(calc, field.source()); - applyOssieDatatype(calc, field.source()); - String exactType = getString(calc, DATA_TYPE); - if (exactType != null && !SalesforceDataTypeMapper.areCompatible(binding.datatype(), exactType)) { - throw new IllegalArgumentException("Field '" + datasetName + "." + field.name() - + "' calculated datatype conflicts with Salesforce extension dataType '" + exactType + "'"); - } - calc.putIfAbsent(DATA_TYPE, SalesforceDataTypeMapper.toSalesforce(binding.datatype())); - applyFieldDefaults(calc); - getOrCreateList(outputData, SEMANTIC_CALCULATED_DIMENSIONS).add(calc); - } + /** + * Maps Ossie dataset fields to Salesforce SemanticDimensions and SemanticMeasurements. + * + * @param outputData The output map containing semanticModel + * @param sourceData The source Ossie data + */ + private void mapOssieToSalesforce( + Map sourceData, Map outputData) { + + List ossieDatasets = getList(sourceData, DATASETS); + + List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); + + for (Object ossieDatasetObj : ossieDatasets) { + Map ossieDataset = asMap(ossieDatasetObj); + + String datasetName = getString(ossieDataset, NAME); + if (datasetName == null) continue; + + // Find matching SemanticDataObject + Map sfDataObject = findItemById(sfDataObjects, API_NAME, datasetName); + if (sfDataObject == null) continue; + + processFieldsForDataset(ossieDataset, sfDataObject, outputData); } } @@ -208,7 +193,7 @@ private Map convertDimensionToOssieField(Map sfD // Wrap dataObjectFieldName in expression structure String dataObjectFieldName = getString(sfDimension, DATA_OBJECT_FIELD_NAME); if (dataObjectFieldName != null) { - ossieField.put(EXPRESSION, wrapPhysicalExpression(dataObjectFieldName)); + ossieField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); } // Store unmapped properties in custom_extensions @@ -227,7 +212,7 @@ private Map convertMeasurementToOssieField(Map s String dataObjectFieldName = getString(sfMeasurement, DATA_OBJECT_FIELD_NAME); if (dataObjectFieldName != null) { - ossieField.put(EXPRESSION, wrapPhysicalExpression(dataObjectFieldName)); + ossieField.put(EXPRESSION, wrapExpression(dataObjectFieldName)); } // Store unmapped properties in custom_extensions @@ -278,12 +263,83 @@ private Map wrapExpression(String expressionValue) { return expression; } - /** Physical columns are SQL identifiers, even when they contain operators or spaces. */ - private Map wrapPhysicalExpression(String column) { - return Map.of(DIALECTS, List.of(Map.of(DIALECT, "ANSI_SQL", EXPRESSION, - "\"" + column.replace("\"", "\"\"") + "\""))); + /** + * Processes all fields for a single dataset. + * + *

Routing Logic: + * + * + * + * + * + *
Expression TypeHas dimension?Routes To
DirectYesdataObject.semanticDimensions
DirectNodataObject.semanticMeasurements
CalculatedN/AMODEL.semanticCalculatedDimensions
+ * + * @param ossieDataset The Ossie dataset + * @param sfDataObject The Salesforce data object to add direct fields to + * @param outputData The Salesforce model for adding calculated dimensions + */ + private void processFieldsForDataset( + Map ossieDataset, Map sfDataObject, Map outputData) { + List ossieFields = getList(ossieDataset, FIELDS); + if (ossieFields == null) { + return; + } + + List sfDimensions = getList(sfDataObject, SEMANTIC_DIMENSIONS); + List sfMeasurements = getList(sfDataObject, SEMANTIC_MEASUREMENTS); + + for (Object ossieFieldObj : ossieFields) { + Map ossieField = asMap(ossieFieldObj); + + // Determine field type based on Ossie structure + boolean hasDimension = ossieField.containsKey(DIMENSION); + ExpressionInfo expressionInfo = unwrapExpression(ossieField); + + String expression = expressionInfo.expression(); + String dialect = expressionInfo.dialect(); + + // Skip calculated fields for non-Tableau dialects till we agree on a common dialect. + if (!DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression)) { + continue; + } + + // Check if this is a calculated field (Tableau dialect with calculated expression) + boolean isCalculated = DIALECT_TABLEAU.equals(dialect) && isCalculatedExpression(expression); + + if (isCalculated) { + // Create a semantic calculated dimension + Map calcDim = createSemanticCalculatedDimension(ossieField, expression); + + customExtensionHandler.restoreSalesforceCustomExtension(calcDim, ossieField); + + applyOssieDatatype(calcDim, ossieField); + + applyFieldDefaults(calcDim); + + // Add to semantic calculated dimensions array + List calcDimensions = getOrCreateList(outputData, SEMANTIC_CALCULATED_DIMENSIONS); + calcDimensions.add(calcDim); + } else { + // Non calculated field - add to data object + FieldType fieldType = hasDimension? FieldType.DIMENSION : FieldType.MEASUREMENT; + + Map sfField = mapFieldProperties(ossieField, expression); + + customExtensionHandler.restoreSalesforceCustomExtension(sfField, ossieField); + + applyOssieDatatype(sfField, ossieField); + + applyFieldDefaults(sfField); + + RoutingResult result = + routeFieldToArray(sfField, fieldType, sfDataObject, sfDimensions, sfMeasurements); + sfDimensions = result.dataObjectDimensions(); + sfMeasurements = result.dataObjectMeasurements(); + } + } } + /** * Maps field properties based on whether the field is calculated. * Includes common properties plus type-specific properties. @@ -344,12 +400,143 @@ private Map createSemanticCalculatedDimension( } // Set syntax for Tableau expressions - calcDim.put("syntax", "Tua"); - calcDim.put("level", "Row"); + calcDim.put("syntax", DIALECT_TABLEAU); return calcDim; } + /** + * Routes a field to the appropriate array based on its type. + * Initializes arrays lazily using computeIfAbsent. + * + * @param sfField The Salesforce field to route + * @param fieldType The field type + * @param sfDataObject The data object (for data object-level arrays) + * @return Updated arrays for all levels + */ + private RoutingResult routeFieldToArray( + Map sfField, + FieldType fieldType, + Map sfDataObject, + List currentDataObjectDimensions, + List currentDataObjectMeasurements) { + + List dataObjectDimensions = currentDataObjectDimensions; + List dataObjectMeasurements = currentDataObjectMeasurements; + + switch (fieldType) { + case DIMENSION: + dataObjectDimensions = getOrCreateList(sfDataObject, SEMANTIC_DIMENSIONS); + dataObjectDimensions.add(sfField); + break; + + case MEASUREMENT: + dataObjectMeasurements = getOrCreateList(sfDataObject, SEMANTIC_MEASUREMENTS); + dataObjectMeasurements.add(sfField); + break; + } + + return new RoutingResult(dataObjectDimensions, dataObjectMeasurements); + } + + /** + * Helper record to return updated data object arrays. + */ + private record RoutingResult(List dataObjectDimensions, List dataObjectMeasurements) {} + + /** + * Helper record to return expression value along with its dialect type. + */ + private record ExpressionInfo(String expression, String dialect) {} + + /** + * Extracts the expression value and dialect from Ossie field's expression.dialects[0].expression. + * This unwraps the nested structure to get the simple column reference and its dialect. + * + * @param ossieField The Ossie field containing expression structure + * @return ExpressionInfo containing the expression string and dialect type, or null if not found + */ + private ExpressionInfo unwrapExpression(Map ossieField) { + Object expressionObj = ossieField.get(EXPRESSION); + + Map expression = asMap(expressionObj); + Object dialectsObj = expression.get(DIALECTS); + + List dialects = asList(dialectsObj); + + Object selectedDialectObj = null; + for (Object dialectObj : dialects) { + Map dialect = asMap(dialectObj); + String dialectType = getString(dialect, DIALECT); + if (DIALECT_TABLEAU.equals(dialectType)) { + selectedDialectObj = dialectObj; + break; + } + } + + if (selectedDialectObj == null) { + selectedDialectObj = dialects.get(0); + } + + Map selectedDialect = asMap(selectedDialectObj); + Object expressionValue = selectedDialect.get(EXPRESSION); + String dialectType = getString(selectedDialect, DIALECT); + + return new ExpressionInfo((String) expressionValue, dialectType); + } + + /** + * Determines if an expression is calculated or a direct column reference. + * + *

A calculated expression contains: + *

    + *
  • SQL functions: CONCAT(), SUM(), CAST(), etc.
    • + *
  • Operators: +, -, *, /, %, ||
    • + *
  • SQL keywords: CASE, WHEN, AND, OR, etc.
    • + *
  • Comparisons: {@literal >, <, =, !=, <>}
    • + *
+ * + *

A direct reference is a simple column name (possibly table-qualified): + *

    + *
  • customer_name
    • + *
  • customers.customer_name
    • + *
  • schema.table.column
    • + *
+ * + * @param expression The SQL expression to evaluate + * @return true if calculated, false if direct reference + */ + private boolean isCalculatedExpression(String expression) { + if (expression == null || expression.isEmpty()) { + return false; + } + + String normalized = expression.trim().toUpperCase(); + + // Check for function calls (presence of parentheses) + if (normalized.contains("(") || normalized.contains("[")) { + return true; + } + + // Check for operators (arithmetic, comparison, string concatenation) + if (normalized.contains("*") || normalized.contains("/") || normalized.contains("%") || + normalized.contains("||") || normalized.contains("::") || + normalized.contains(">") || normalized.contains("<") || + normalized.contains("!=") || normalized.contains("<>")) { + return true; + } + + // Check for arithmetic/comparison operators with spaces (avoid false positives like "customer-id") + if (normalized.contains(" + ") || normalized.contains(" - ") || + normalized.contains(" * ") || normalized.contains(" / ") || + normalized.contains(" = ")) { + return true; + } + + // Check for SQL keywords using compiled pattern + return CALCULATED_KEYWORDS_PATTERN.matcher(normalized).find(); + } + /** * Applies default values for required Salesforce field properties. * Only sets defaults if the property is not already present. @@ -385,9 +572,12 @@ private void applyOssieDatatype(Map sfField, Map if (exactSalesforceDataType != null) { if (ossieDatatype != null && !SalesforceDataTypeMapper.areCompatible(ossieDatatype, exactSalesforceDataType)) { - throw new IllegalArgumentException("Field '" + getString(ossieField, NAME) - + "' has Ossie datatype '" + ossieDatatype - + "' that conflicts with Salesforce extension dataType '" + exactSalesforceDataType + "'"); + logger.warn( + "Field '{}' has Ossie datatype '{}' that conflicts with exact Salesforce dataType '{}'; " + + "preserving the Salesforce extension value", + getString(ossieField, NAME), + ossieDatatype, + exactSalesforceDataType); } return; } @@ -397,9 +587,11 @@ private void applyOssieDatatype(Map sfField, Map } if (mappedSalesforceDataType == null) { - throw new IllegalArgumentException("Field '" + getString(ossieField, NAME) - + "' has Ossie datatype '" + ossieDatatype - + "' with no safe Salesforce mapping; provide a compatible native type"); + logger.warn( + "Field '{}' has Ossie datatype '{}' with no safe Salesforce mapping; omitting dataType", + getString(ossieField, NAME), + ossieDatatype); + return; } sfField.put(DATA_TYPE, mappedSalesforceDataType); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java deleted file mode 100644 index 70bc1fd1..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricCompilationPlan.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.util.DataStructureUtils.*; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.ossie.exception.ConversionException; - -/** Per-model metric dependency plan. References are checked before aggregate formulas are inlined. */ -final class MetricCompilationPlan { - private static final int MAX_DEPENDENCY_DEPTH = 128; - private final MetricFieldResolver fields; - private final Map> declarations = new LinkedHashMap<>(); - private final Map> sqlNames = new LinkedHashMap<>(); - private final Map compiled = new LinkedHashMap<>(); - private final Set visiting = new LinkedHashSet<>(); - - MetricCompilationPlan(Map source, MetricFieldResolver fields) { - this.fields = fields; - List metrics = getList(source, "metrics"); - if (metrics == null) return; - for (Object value : metrics) { - Map metric = asMap(value); - String name = getString(metric, "name"); - if (name == null || name.isBlank()) throw new ConversionException("Metric name must not be empty"); - if (declarations.putIfAbsent(name, metric) != null) { - throw new ConversionException("Metric '" + name + "': duplicate metric name"); - } - sqlNames.computeIfAbsent(MetricFieldResolver.normalizeDeclaration(name), key -> new ArrayList<>()).add(name); - } - } - - ExpressionCompiler.Compiled compile(String name) { - ExpressionCompiler.Compiled cached = compiled.get(name); - if (cached != null) return cached; - if (visiting.contains(name)) { - throw new ConversionException("Metric dependency cycle: " + String.join(" -> ", visiting) + " -> " + name); - } - if (visiting.size() >= MAX_DEPENDENCY_DEPTH) { - throw new ConversionException("Metric '" + name + "': dependency depth exceeds " + MAX_DEPENDENCY_DEPTH); - } - Map metric = declarations.get(name); - if (metric == null) throw new ConversionException("Unknown metric '" + name + "'"); - visiting.add(name); - try { - ExpressionCompiler.Compiled result = MetricExpressionTranslator.compile(metric, this::resolve); - try { fields.validateDatasets(result.datasets()); } - catch (IllegalArgumentException e) { throw new ConversionException("Metric '" + name + "': " + e.getMessage(), e); } - compiled.put(name, result); - return result; - } finally { - visiting.remove(name); - } - } - - private ExpressionCompiler.Binding resolve(ExpressionCompiler.Reference reference) { - if (reference.parts().size() == 1) { - MetricFieldResolver.Identifier identifier = reference.parts().get(0); - List matches = reference.tableau() - ? (declarations.containsKey(identifier.text()) ? List.of(identifier.text()) : List.of()) - : sqlNames.getOrDefault(MetricFieldResolver.normalize(identifier), List.of()); - if (!matches.isEmpty()) { - if (matches.size() != 1 || fields.hasUnqualifiedField(identifier, reference.tableau())) { - throw new IllegalArgumentException("ambiguous field or metric reference '" + identifier.text() - + "'; qualify the field or use a unique metric name"); - } - ExpressionCompiler.Compiled metric = compile(matches.get(0)); - return new ExpressionCompiler.Binding("(" + metric.expression() + ")", metric.datatype(), - metric.datasets(), metric.level()); - } - } - return fields.resolveBinding(reference); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpression.java similarity index 84% rename from converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpression.java index 7091029d..9d84ca9b 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ExpressionAst.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpression.java @@ -24,11 +24,13 @@ import java.util.Set; /** Immutable compiler nodes. Neither source parsing nor type checking emits target text. */ -final class ExpressionAst { - private ExpressionAst() {} +final class MetricExpression { + private MetricExpression() {} sealed interface Node permits Literal, Field, Unary, Binary, Call, Conditional {} record Literal(Object value) implements Node {} - record Field(ExpressionCompiler.Reference reference) implements Node {} + record Field(List parts, boolean tableau) implements Node { + Field { parts = List.copyOf(parts); } + } record Unary(String operator, Node operand) implements Node {} record Binary(String operator, Node left, Node right) implements Node {} record Call(String name, List arguments, boolean distinct) implements Node { @@ -38,6 +40,7 @@ record Call(String name, List arguments, boolean distinct) implements Node record Conditional(List branches, Node otherwise) implements Node { Conditional { branches = List.copyOf(branches); } } + enum Level { CONSTANT, ROW, AGGREGATE } enum Type { INTEGER("Integer"), DECIMAL("Decimal"), FLOAT("Float"), STRING("String"), BOOLEAN("Boolean"), DATE("Date"), DATETIME("DateTime"), DATETIME_TZ("DateTimeTz"), @@ -51,8 +54,8 @@ static Type of(String datatype) { return UNKNOWN; } } - record Typed(Node node, Type type, ExpressionCompiler.Level level, Set datasets, - List children, ExpressionCompiler.Binding binding, BigDecimal number) { + record Typed(Node node, Type type, Level level, Set datasets, + List children, MetricFieldResolver.ResolvedField binding, BigDecimal number) { Typed { datasets = Set.copyOf(datasets); children = List.copyOf(children); } } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java index 25360f3f..33160177 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricExpressionTranslator.java @@ -19,12 +19,21 @@ package org.apache.ossie.converter; -import static org.apache.ossie.util.DataStructureUtils.getString; +import static org.apache.ossie.converter.MetricExpression.*; +import static org.apache.ossie.util.DataStructureUtils.*; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.ossie.exception.ConversionException; -/** Applies measurement-specific constraints around the shared expression compiler. */ +/** Compiles the documented metric subset: parse, bind/check, then emit bounded Tua text. */ final class MetricExpressionTranslator { + private static final List DIALECTS = List.of("TABLEAU", "SNOWFLAKE", "ANSI_SQL"); + private static final Set AGGREGATES = Set.of("SUM", "AVG", "MIN", "MAX", "COUNT", "COUNTD"); record Result(String expression, String dataType) {} private MetricExpressionTranslator() {} @@ -34,45 +43,287 @@ static Result translate(Map metric, Map sourceMo } static Result translate(Map metric, MetricFieldResolver resolver) { - ExpressionCompiler.Compiled result = compile(metric, resolver::resolveBinding); - try { - resolver.validateDatasets(result.datasets()); - } catch (IllegalArgumentException e) { - throw new ConversionException("Metric '" + getString(metric, "name") + "': " + e.getMessage(), e); - } - return new Result(result.expression(), "Number"); - } - - static ExpressionCompiler.Compiled compile(Map metric, - ExpressionCompiler.ReferenceResolver references) { String name = getString(metric, "name"); try { - ExpressionCompiler.Selected selected = ExpressionCompiler.select(metric); - ExpressionCompiler.Compiled result = ExpressionCompiler.compile( - ExpressionCompiler.parse(selected.text(), selected.dialect()), references); - ExpressionAst.Type actual = result.datatype() == null ? ExpressionAst.Type.NULL : ExpressionAst.Type.of(result.datatype()); - ExpressionAst.Type declared = ExpressionAst.Type.of(getString(metric, "datatype")); - if (metric.containsKey("datatype") && declared == ExpressionAst.Type.UNKNOWN) { + Map expression = getMap(metric, "expression"); + List dialects = expression == null ? null : getList(expression, "dialects"); + if (dialects == null) { + throw new IllegalArgumentException("missing expression.dialects; provide TABLEAU, SNOWFLAKE or ANSI_SQL"); + } + Map candidates = new java.util.LinkedHashMap<>(); + for (Object entry : dialects) { + Map value = asMap(entry); + String dialect = getString(value, "dialect"); + if (DIALECTS.contains(dialect)) { + if (candidates.containsKey(dialect)) { + throw new IllegalArgumentException("ambiguous expression: multiple " + dialect + " entries"); + } + candidates.put(dialect, getString(value, "expression")); + } + } + String dialect = DIALECTS.stream().filter(candidates::containsKey).findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "no supported dialect; provide TABLEAU, SNOWFLAKE or ANSI_SQL")); + String text = candidates.get(dialect); + if (text == null || text.isBlank()) { + throw new IllegalArgumentException(dialect + " expression is empty"); + } + Node parsed = dialect.equals("TABLEAU") + ? new TuaMetricExpressionParser(TuaMetricExpressionParser.tokenize(text, dialect)).parse() : SqlMetricExpressionParser.parse(text, dialect); + Typed result = new Analyzer(dialect, resolver).analyze(parsed); + resolver.validateDatasets(result.datasets()); + Type declared = Type.of(getString(metric, "datatype")); + if (metric.containsKey("datatype") && declared == Type.UNKNOWN) { throw new IllegalArgumentException("unsupported metric datatype " + getString(metric, "datatype")); } - if (!actual.numeric() && actual != ExpressionAst.Type.NULL) { - throw new IllegalArgumentException("calculated measurements must be numeric, found " + actual); + if (!result.type().numeric() && result.type() != Type.NULL) { + throw new IllegalArgumentException("calculated measurements must be numeric, found " + result.type()); } - if (declared != ExpressionAst.Type.UNKNOWN && (!declared.numeric() - || declared == ExpressionAst.Type.INTEGER && actual != ExpressionAst.Type.INTEGER && actual != ExpressionAst.Type.NULL)) { - throw new IllegalArgumentException("datatype " + getString(metric, "datatype") + " is incompatible with expression result " + actual); + if (declared != Type.UNKNOWN && (!declared.numeric() + || (declared == Type.INTEGER && result.type() != Type.INTEGER && result.type() != Type.NULL))) { + throw new IllegalArgumentException("datatype " + getString(metric, "datatype") + + " is incompatible with expression result " + result.type()); } - if (actual == ExpressionAst.Type.NULL && !declared.numeric()) { + if (result.type() == Type.NULL && !declared.numeric()) { throw new IllegalArgumentException("all-null result needs an explicit numeric datatype"); } - if (result.level() == ExpressionCompiler.Level.ROW) { + if (result.level() == Level.ROW) { throw new IllegalArgumentException("unaggregated field in metric; use an explicit aggregate"); } - return actual == ExpressionAst.Type.NULL - ? new ExpressionCompiler.Compiled(result.expression(), declared.datatype, result.level(), result.datasets()) - : result; + return new Result(new Emitter().emit(result), "Number"); } catch (IllegalArgumentException e) { throw new ConversionException("Metric '" + name + "': " + e.getMessage(), e); } } + + /** Checks the complete tree before any target text is emitted. */ + private static final class Analyzer { + private final String dialect; + private final MetricFieldResolver resolver; + private int depth; + Analyzer(String dialect, MetricFieldResolver resolver) { + this.dialect = dialect; this.resolver = resolver; + } + Typed analyze(Node node) { + if (++depth > 128) throw new IllegalArgumentException("expression nesting exceeds 128 levels"); + try { return analyzeNode(node); } finally { depth--; } + } + private Typed analyzeNode(Node node) { + if (node instanceof Literal literal) { + Object value = literal.value(); + Type type = value == null ? Type.NULL : value instanceof Boolean ? Type.BOOLEAN + : value instanceof String ? Type.STRING + : ((BigDecimal) value).stripTrailingZeros().scale() <= 0 ? Type.INTEGER : Type.DECIMAL; + return new Typed(node, type, Level.CONSTANT, Set.of(), List.of(), null, + value instanceof BigDecimal number ? number : null); + } + if (node instanceof Field field) { + MetricFieldResolver.ResolvedField binding = resolver.resolve(field.parts(), field.tableau()); + if (binding == null || binding.expression() == null || binding.expression().isBlank()) { + throw new IllegalArgumentException("field resolver returned no binding"); + } + Type type = Type.of(binding.datatype()); + if (type == Type.UNKNOWN) throw new IllegalArgumentException("field reference needs known field datatypes"); + return new Typed(node, type, Level.ROW, Set.of(binding.dataset()), List.of(), binding, null); + } + if (node instanceof Unary unary) { + Typed child = analyze(unary.operand()); + String operator = unary.operator(); + Type result = child.type(); + BigDecimal number = child.number(); + if (operator.equals("ISNULL")) { result = Type.BOOLEAN; number = null; } + else if (operator.equals("NOT")) { require(child, Type.BOOLEAN, "NOT"); result = Type.BOOLEAN; number = null; } + else { numeric(child, "unary " + operator); if (operator.equals("-") && number != null) number = number.negate(); } + return new Typed(node, result, child.level(), child.datasets(), List.of(child), null, number); + } + if (node instanceof Binary binary) { + Typed left = analyze(binary.left()); Typed right = analyze(binary.right()); + String op = binary.operator(); + Type result; + if (op.equals("AND") || op.equals("OR")) { + require(left, Type.BOOLEAN, op); require(right, Type.BOOLEAN, op); result = Type.BOOLEAN; + } else if (Set.of("=", "!=", "<", "<=", ">", ">=").contains(op)) { + compatible(left.type(), right.type(), "comparison"); + if (!Set.of("=", "!=").contains(op) && (left.type() == Type.BOOLEAN || right.type() == Type.BOOLEAN)) { + throw new IllegalArgumentException("ordered comparison requires numeric, text or temporal operands"); + } + result = Type.BOOLEAN; + } else { + numeric(left, op); numeric(right, op); + result = compatible(left.type(), right.type(), op); + if (op.equals("/")) { + if (right.number() != null && right.number().signum() == 0) { + throw new IllegalArgumentException("division by literal zero; use NULLIF(denominator, 0) for a nullable denominator"); + } + result = Type.DECIMAL; + } + } + return compose(node, result, List.of(left, right)); + } + if (node instanceof Conditional conditional) { + List children = new ArrayList<>(); + Type result = Type.NULL; + for (int i = 0; i < conditional.branches().size(); i += 2) { + Typed predicate = analyze(conditional.branches().get(i)); + require(predicate, Type.BOOLEAN, "conditional predicate"); + Typed branch = analyze(conditional.branches().get(i + 1)); + result = compatible(result, branch.type(), "conditional branches"); + children.add(predicate); children.add(branch); + } + Typed otherwise = analyze(conditional.otherwise()); + result = compatible(result, otherwise.type(), "conditional branches"); + children.add(otherwise); + return compose(node, result, children); + } + Call call = (Call) node; + String name = call.name(); + if (call.distinct() && (!name.equals("COUNT") || dialect.equals("TABLEAU"))) { + throw new IllegalArgumentException("DISTINCT is supported only by SQL COUNT(DISTINCT field)"); + } + if (Set.of("COALESCE", "NULLIF", "CEIL").contains(name) && dialect.equals("TABLEAU") + || Set.of("IFNULL", "ISNULL", "CEILING", "COUNTD").contains(name) && !dialect.equals("TABLEAU")) { + throw new IllegalArgumentException(name + " is outside the supported " + dialect + " subset"); + } + int maximum = switch (name) { + case "COALESCE" -> Integer.MAX_VALUE; + case "IFNULL", "NULLIF", "ROUND" -> 2; + case "SUM", "AVG", "MIN", "MAX", "COUNT", "COUNTD", "ISNULL", "ABS", "CEIL", "CEILING", "FLOOR" -> 1; + default -> throw new IllegalArgumentException("unsupported function " + name); + }; + int minimum = Set.of("COALESCE", "IFNULL", "NULLIF").contains(name) ? 2 : 1; + if (call.arguments().size() < minimum || call.arguments().size() > maximum) { + throw new IllegalArgumentException(name + " expects " + + (minimum == maximum ? minimum : minimum + " to " + maximum) + " arguments"); + } + List arguments = call.arguments().stream().map(this::analyze).toList(); + if (AGGREGATES.contains(name)) return aggregate(call, arguments); + return switch (name) { + case "COALESCE", "IFNULL" -> { + Type result = Type.NULL; + for (Typed argument : arguments) result = compatible(result, argument.type(), name + " arguments"); + yield compose(node, result, arguments); + } + case "NULLIF" -> { + compatible(arguments.get(0).type(), arguments.get(1).type(), "NULLIF arguments"); + yield compose(node, arguments.get(0).type(), arguments); + } + case "ISNULL" -> compose(node, Type.BOOLEAN, arguments); + default -> numericFunction(call, arguments); + }; + } + + private Typed aggregate(Call call, List arguments) { + String name = call.name(); Typed argument = arguments.get(0); + if (argument.level() == Level.AGGREGATE) throw new IllegalArgumentException("nested aggregate " + name + " is unsupported"); + if (argument.datasets().isEmpty()) throw new IllegalArgumentException(name + " needs a declared field to establish its dataset"); + if (argument.datasets().size() > 1) throw new IllegalArgumentException("one aggregate cannot combine fields from multiple datasets"); + boolean count = name.equals("COUNT") || name.equals("COUNTD"); + if (count && !(argument.node() instanceof Field)) { + throw new IllegalArgumentException(name + " requires a declared field; counting expressions is unsupported"); + } + if (name.equals("MIN") || name.equals("MAX")) { + if (argument.type() == Type.BOOLEAN || argument.type() == Type.UNKNOWN) { + throw new IllegalArgumentException(name + " requires numeric, text or temporal operands"); + } + } else if (!count) numeric(argument, name); + Type result = count ? Type.INTEGER : name.equals("AVG") ? Type.DECIMAL : argument.type(); + return new Typed(call, result, Level.AGGREGATE, argument.datasets(), arguments, null, null); + } + private Typed numericFunction(Call call, List arguments) { + Typed value = arguments.get(0); numeric(value, call.name()); + BigDecimal places = BigDecimal.ZERO; + if (arguments.size() == 2) { + places = arguments.get(1).number(); + if (places == null || places.stripTrailingZeros().scale() > 0 + || places.compareTo(BigDecimal.valueOf(Integer.MIN_VALUE)) < 0 + || places.compareTo(BigDecimal.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException("ROUND precision must be a 32-bit integer literal"); + } + } + Type result = value.type(); + if (result != Type.NULL && (Set.of("CEIL", "CEILING", "FLOOR").contains(call.name()) + || call.name().equals("ROUND") && places.signum() <= 0)) result = Type.INTEGER; + return compose(call, result, arguments); + } + private Typed compose(Node node, Type type, List arguments) { + Level level = Level.CONSTANT; Set datasets = new HashSet<>(); + for (Typed argument : arguments) { + if (level != Level.CONSTANT && argument.level() != Level.CONSTANT && level != argument.level()) { + throw new IllegalArgumentException("cannot mix aggregate and unaggregated field expressions"); + } + if (argument.level() != Level.CONSTANT) level = argument.level(); + datasets.addAll(argument.datasets()); + } + return new Typed(node, type, level, datasets, arguments, null, null); + } + private static void numeric(Typed value, String context) { + if (!value.type().numeric() && value.type() != Type.NULL) { + throw new IllegalArgumentException(context + " requires numeric operands, found " + value.type() + "; declare a compatible field datatype"); + } + } + private static void require(Typed value, Type expected, String context) { + if (value.type() != expected && value.type() != Type.NULL) throw new IllegalArgumentException(context + " requires " + expected + ", found " + value.type()); + } + private static Type compatible(Type left, Type right, String context) { + if (left == Type.UNKNOWN || right == Type.UNKNOWN) throw new IllegalArgumentException(context + " needs known field datatypes"); + if (left == Type.NULL) return right; + if (right == Type.NULL || left == right) return left; + if (left.numeric() && right.numeric()) return left == Type.FLOAT || right == Type.FLOAT ? Type.FLOAT : Type.DECIMAL; + throw new IllegalArgumentException(context + " has incompatible types " + left + " and " + right); + } + } + + /** Streams checked nodes so NULLIF expansion cannot allocate an unbounded string. */ + private static final class Emitter { + private static final int MAX_OUTPUT = 131072; + private final StringBuilder output = new StringBuilder(); + String emit(Typed expression) { append(expression); return output.toString(); } + private void text(String text) { + if ((long) output.length() + text.length() > MAX_OUTPUT) { + throw new IllegalArgumentException("translated expression exceeds 131072 characters"); + } + output.append(text); + } + private void append(Typed value) { + Node node = value.node(); List children = value.children(); + if (node instanceof Literal literal) { + Object content = literal.value(); + text(content == null ? "NULL" : content instanceof String string ? "'" + string.replace("'", "''") + "'" + : content instanceof Boolean bool ? bool ? "TRUE" : "FALSE" : ((BigDecimal) content).toPlainString()); + } else if (node instanceof Field) { + text(value.binding().expression()); + } else if (node instanceof Unary unary) { + switch (unary.operator()) { + case "+" -> append(children.get(0)); + case "-" -> { text("(-"); append(children.get(0)); text(")"); } + case "NOT" -> { text("(NOT "); append(children.get(0)); text(")"); } + case "ISNULL" -> { text("ISNULL("); append(children.get(0)); text(")"); } + default -> throw new IllegalStateException("unvalidated unary operator"); + } + } else if (node instanceof Binary binary) { + text("("); append(children.get(0)); text(" " + binary.operator() + " "); append(children.get(1)); text(")"); + } else if (node instanceof Conditional) { + text("(IF "); + for (int i = 0; i < children.size() - 1; i += 2) { + if (i > 0) text(" ELSEIF "); + append(children.get(i)); text(" THEN "); append(children.get(i + 1)); + } + text(" ELSE "); append(children.get(children.size() - 1)); text(" END)"); + } else { + Call call = (Call) node; + if (call.name().equals("COALESCE") || call.name().equals("IFNULL")) { + for (int i = 0; i < children.size() - 1; i++) { text("IFNULL("); append(children.get(i)); text(", "); } + append(children.get(children.size() - 1)); + for (int i = 0; i < children.size() - 1; i++) text(")"); + } else if (call.name().equals("NULLIF")) { + text("(IF ("); append(children.get(0)); text(" = "); append(children.get(1)); + text(") THEN NULL ELSE "); append(children.get(0)); text(" END)"); + } else { + text((call.distinct() ? "COUNTD" : (call.name().equals("CEIL") ? "CEILING" : call.name())) + "("); + for (int i = 0; i < children.size(); i++) { if (i > 0) text(", "); append(children.get(i)); } + text(")"); + } + } + } + } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java index 2cff9226..46faf463 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricFieldResolver.java @@ -19,195 +19,269 @@ package org.apache.ossie.converter; -import static org.apache.ossie.util.DataStructureUtils.*; +import static org.apache.ossie.util.DataStructureUtils.getList; +import static org.apache.ossie.util.DataStructureUtils.getString; +import static org.apache.ossie.util.DataStructureUtils.streamMaps; -import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.ArrayDeque; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; -/** Model-scoped indexes bind metric references to validated exported semantic identities. */ +/** Binds metric references to declared fields that the Salesforce converter actually exported. */ final class MetricFieldResolver { + record Identifier(String text, boolean quoted) {} + record ResolvedField(String expression, String datatype, String dataset) {} - private record Field(String dataset, String name, Map properties) {} - private final NameIndex> datasets = new NameIndex<>(); - private final NameIndex unqualifiedFields = new NameIndex<>(); - private final Map> fieldsByDataset = new HashMap<>(); - private final Map>> targetDatasets; - private final Map>>> targetFields = new HashMap<>(); - private final Map>> targetCalculatedFields; - private final Map components; - private final FieldExpressionPlan fieldPlan; + private record Field(Map dataset, Map field) {} + + private final List> datasets; + private final List> targetDatasets; + private final List> relationships; + private final List> declaredRelationships; + private record Reference(List parts, boolean tableau) { + Reference { parts = List.copyOf(parts); } + } + private final Map resolved = new HashMap<>(); + private final Map> reachable = new HashMap<>(); + private final List invalidRelationships = new ArrayList<>(); + private Map> graph; MetricFieldResolver(Map sourceModel, Map targetModel) { - this(sourceModel, targetModel, null); - } - - MetricFieldResolver(Map sourceModel, Map targetModel, - FieldExpressionPlan fieldPlan) { - this.fieldPlan = fieldPlan; - for (Map dataset : items(sourceModel, "datasets")) { - String datasetName = getString(dataset, "name"); - datasets.add(datasetName, dataset); - NameIndex fields = fieldsByDataset.computeIfAbsent(datasetName, ignored -> new NameIndex<>()); - for (Map properties : items(dataset, "fields")) { - String name = getString(properties, "name"); - Field field = new Field(datasetName, name, properties); - fields.add(name, field); - unqualifiedFields.add(name, field); - } - } - List> targets = items(targetModel, "semanticDataObjects"); - targetDatasets = index(targets, item -> getString(item, "apiName")); - for (Map dataset : targets) { - List> fields = new ArrayList<>(items(dataset, "semanticDimensions")); - fields.addAll(items(dataset, "semanticMeasurements")); - targetFields.put(getString(dataset, "apiName"), index(fields, item -> getString(item, "apiName"))); - } - targetCalculatedFields = index(items(targetModel, "semanticCalculatedDimensions"), - item -> getString(item, "apiName")); - components = connectedComponents(targetDatasets.keySet(), items(targetModel, "semanticRelationships")); + datasets = items(sourceModel, "datasets"); + targetDatasets = items(targetModel, "semanticDataObjects"); + relationships = items(targetModel, "semanticRelationships"); + declaredRelationships = items(sourceModel, "relationships"); } - /** Connectivity is a prerequisite; it is not a proof of native join-grain equivalence. */ + /** Only unchanged, enabled edges can establish a metric's dataset connectivity. */ void validateDatasets(Set referenced) { if (referenced.size() < 2) return; - Integer component = components.get(referenced.iterator().next()); - if (component == null || referenced.stream().anyMatch(name -> !component.equals(components.get(name)))) { + if (graph == null) graph = validatedGraph(); + String first = referenced.stream().sorted().findFirst().orElseThrow(); + Set visited = reachable.computeIfAbsent(first, this::reachableFrom); + if (!visited.containsAll(referenced)) { throw new IllegalArgumentException("Metric references disconnected datasets " + referenced.stream().sorted().collect(Collectors.joining(", ")) - + "; declare supported relationships connecting them before exporting the metric"); + + "; declare supported relationships connecting them before exporting the metric" + + (invalidRelationships.isEmpty() ? "" : "; unusable relationships: " + String.join("; ", invalidRelationships))); } } - boolean hasUnqualifiedField(Identifier identifier, boolean tableau) { - return !unqualifiedFields.find(identifier, tableau).isEmpty(); + private Map> validatedGraph() { + Map> result = new HashMap<>(); + for (Map dataset : targetDatasets) { + String name = getString(dataset, "apiName"); + if (name != null) result.put(name, new HashSet<>()); + } + for (Map relationship : relationships) { + if (!Boolean.TRUE.equals(relationship.get("isEnabled"))) continue; + String name = getString(relationship, "apiName"); + String left = getString(relationship, "leftSemanticDefinitionApiName"); + String right = getString(relationship, "rightSemanticDefinitionApiName"); + try { + if (!result.containsKey(left) || !result.containsKey(right)) { + throw new IllegalArgumentException("missing exported endpoint"); + } + validateRelationship(relationship, name, left, right); + result.get(left).add(right); + result.get(right).add(left); + } catch (IllegalArgumentException e) { + invalidRelationships.add("'" + name + "': " + e.getMessage()); + } + } + return result; } - ExpressionCompiler.Binding resolveBinding(ExpressionCompiler.Reference reference) { - return binding(reference.parts(), reference.tableau()); + private void validateRelationship(Map target, String name, String left, String right) { + List> matches = declaredRelationships.stream() + .filter(source -> name != null && name.equals(getString(source, "name"))).toList(); + if (matches.size() != 1) throw new IllegalArgumentException("expected one source relationship"); + Map source = matches.get(0); + if (!left.equals(getString(source, "from")) || !right.equals(getString(source, "to"))) { + throw new IllegalArgumentException("changed endpoints"); + } + List from = getList(source, "from_columns"); + List to = getList(source, "to_columns"); + List> criteria = items(target, "criteria"); + if (from == null || to == null || from.isEmpty() || from.size() != to.size() || from.size() != criteria.size()) { + throw new IllegalArgumentException("changed or missing composite join keys"); + } + for (int i = 0; i < from.size(); i++) { + Map pair = criteria.get(i); + if (!(from.get(i) instanceof String leftKey) || !(to.get(i) instanceof String rightKey) + || !leftKey.equals(pair.get("leftSemanticFieldApiName")) + || !rightKey.equals(pair.get("rightSemanticFieldApiName"))) { + throw new IllegalArgumentException("changed join key correspondence"); + } + for (String side : List.of("leftFieldType", "rightFieldType")) { + if (pair.containsKey(side) && !"TableField".equals(pair.get(side))) { + throw new IllegalArgumentException("calculated join keys are unsupported"); + } + } + resolve(List.of(new Identifier(left, true), new Identifier(leftKey, true)), true); + resolve(List.of(new Identifier(right, true), new Identifier(rightKey, true)), true); + } + } + + private Set reachableFrom(String start) { + Set visited = new HashSet<>(); + ArrayDeque pending = new ArrayDeque<>(); + pending.add(start); + while (!pending.isEmpty()) { + String dataset = pending.removeFirst(); + if (visited.add(dataset)) pending.addAll(graph.getOrDefault(dataset, Set.of())); + } + return Set.copyOf(visited); } ResolvedField resolve(List parts, boolean tableau) { - ExpressionCompiler.Binding value = binding(parts, tableau); - return new ResolvedField(value.expression(), value.datatype(), value.dataset()); + return resolved.computeIfAbsent(new Reference(parts, tableau), key -> resolveUncached(key.parts(), key.tableau())); } - private ExpressionCompiler.Binding binding(List parts, boolean tableau) { + private ResolvedField resolveUncached(List parts, boolean tableau) { String reference = parts.stream().map(Identifier::text).collect(Collectors.joining(".")); if (parts.isEmpty() || parts.size() > 2) { throw new IllegalArgumentException("Reference '" + reference + "' must name a declared field or dataset.field; physical source paths are unsupported"); } if (tableau && parts.size() != 2) { - throw new IllegalArgumentException("TABLEAU field reference '" + reference + "' must use [dataset].[field]"); + throw new IllegalArgumentException("TABLEAU field reference '" + reference + + "' must use [dataset].[field]"); } - List candidates; + + List> candidates = datasets; if (parts.size() == 2) { - List> matches = datasets.find(parts.get(0), tableau); - if (matches.isEmpty()) throw new IllegalArgumentException("Unknown dataset in reference '" + reference - + "'; use a declared dataset name, not its physical source"); - if (matches.size() > 1) throw new IllegalArgumentException("Ambiguous dataset in reference '" + reference - + "'; dataset declarations must have distinct names"); - candidates = fieldsByDataset.get(getString(matches.get(0), "name")).find(parts.get(1), tableau); - } else { - candidates = unqualifiedFields.find(parts.get(0), tableau); - } - if (candidates.isEmpty()) throw new IllegalArgumentException("Unknown field reference '" + reference - + "'; declare the field under datasets[].fields before exporting the metric"); - if (candidates.size() > 1) throw new IllegalArgumentException("Ambiguous field reference '" + reference - + "'; qualify the dataset and remove duplicate field declarations"); - Field match = candidates.get(0); - if (datasets.declarations(match.dataset, tableau).size() > 1) { - throw new IllegalArgumentException("Ambiguous dataset for reference '" + reference - + "'; dataset declarations must have distinct names"); + candidates = datasets.stream() + .filter(dataset -> matches(parts.get(0), getString(dataset, "name"), tableau)) + .toList(); + if (candidates.isEmpty()) { + throw new IllegalArgumentException("Unknown dataset in reference '" + reference + + "'; use a declared dataset name, not its physical source"); + } + if (candidates.size() > 1) { + throw new IllegalArgumentException("Ambiguous dataset in reference '" + reference + + "'; dataset declarations must have distinct names"); + } } - Map targetDataset = exported(targetDatasets, match.dataset, "dataset", reference); - if (fieldPlan != null && !fieldPlan.isDirect(match.dataset, match.name)) { - Map targetField = exported(targetCalculatedFields, - fieldPlan.calculatedApiName(match.dataset, match.name), "field", reference); - ExpressionCompiler.Binding resolved = fieldPlan.resolve(match.dataset, match.name); - if (!fieldPlan.hasPhysicalDependencies(match.dataset, match.name)) { - throw new IllegalArgumentException("Field reference '" + reference - + "' is a constant row field without a physical dataset anchor; " - + "combine it with a direct field in a derived row expression before using it in a metric"); + + Identifier fieldName = parts.get(parts.size() - 1); + List fields = new ArrayList<>(); + for (Map dataset : candidates) { + for (Map field : items(dataset, "fields")) { + if (matches(fieldName, getString(field, "name"), tableau)) { + fields.add(new Field(dataset, field)); + } } - checkType(resolved.datatype(), getString(targetField, "dataType"), reference); - return resolved; } - Map targetField = exported(targetFields.get(match.dataset), match.name, "field", reference); - String datatype = getString(match.properties, "datatype"); - String targetType = getString(targetField, "dataType"); - if (datatype == null || datatype.isBlank()) datatype = SalesforceDataTypeMapper.toOssie(targetType); - checkType(datatype, targetType, reference); - String datasetName = getString(targetDataset, "apiName"); - return new ExpressionCompiler.Binding(bracket(datasetName) + "." + bracket(getString(targetField, "apiName")), - datatype, datasetName); - } + if (fields.isEmpty()) { + throw new IllegalArgumentException("Unknown field reference '" + reference + + "'; declare the field under datasets[].fields before exporting the metric"); + } + if (fields.size() > 1) { + throw new IllegalArgumentException("Ambiguous field reference '" + reference + + "'; qualify the dataset and remove duplicate field declarations"); + } + + Field match = fields.get(0); + String datasetName = getString(match.dataset(), "name"); + // An unqualified field must not accidentally select one of two equivalent datasets. + if (datasets.stream().filter(dataset -> equivalentDeclaration( + datasetName, getString(dataset, "name"), tableau)).count() > 1) { + throw new IllegalArgumentException("Ambiguous dataset for reference '" + reference + + "'; dataset declarations must have distinct names"); + } + String sourceFieldName = getString(match.field(), "name"); + Map targetDataset = exportedItem(targetDatasets, datasetName, + "dataset", reference); + List> targetFields = new ArrayList<>(items(targetDataset, "semanticDimensions")); + targetFields.addAll(items(targetDataset, "semanticMeasurements")); + Map targetField = exportedItem(targetFields, sourceFieldName, "field", reference); - private static void checkType(String datatype, String targetType, String reference) { + String datatype = getString(match.field(), "datatype"); + String targetType = getString(targetField, "dataType"); + if (datatype == null || datatype.isBlank()) { + datatype = SalesforceDataTypeMapper.toOssie(targetType); + } if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { throw new IllegalArgumentException("Field reference '" + reference + "' has no supported datatype; declare a portable field datatype"); } - if (targetType == null || targetType.isBlank() || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { - throw new IllegalArgumentException("Field reference '" + reference + "' has datatype '" + datatype - + "' but exported Salesforce dataType '" + targetType + "'; use compatible field types"); + if (targetType == null || targetType.isBlank() + || !SalesforceDataTypeMapper.areCompatible(datatype, targetType)) { + throw new IllegalArgumentException("Field reference '" + reference + "' has datatype '" + + datatype + "' but exported Salesforce dataType '" + targetType + + "'; use compatible field types"); } - } - private static Map exported(Map>> index, - String name, String kind, String reference) { - List> matches = index == null ? List.of() : index.getOrDefault(name, List.of()); - if (matches.isEmpty()) throw new IllegalArgumentException("Declared " + kind + " in reference '" + reference - + "' was not exported as a direct Salesforce semantic " + kind - + "; calculated or omitted fields are unsupported in metric references without a compiled field plan"); - if (matches.size() > 1) throw new IllegalArgumentException("Ambiguous exported Salesforce " + kind - + " for reference '" + reference + "'; apiName values must be unique"); - return matches.get(0); + validateDirectBinding(targetField, reference); + String targetDatasetName = getString(targetDataset, "apiName"); + String targetFieldName = getString(targetField, "apiName"); + return new ResolvedField(bracket(targetDatasetName) + "." + bracket(targetFieldName), + datatype, targetDatasetName); } - private static Map connectedComponents(Set names, List> relationships) { - Map> graph = new LinkedHashMap<>(); - names.forEach(name -> graph.put(name, new HashSet<>())); - for (Map relationship : relationships) { - if (Boolean.FALSE.equals(relationship.get("isEnabled"))) continue; - String left = getString(relationship, "leftSemanticDefinitionApiName"); - String right = getString(relationship, "rightSemanticDefinitionApiName"); - if (graph.containsKey(left) && graph.containsKey(right)) { - graph.get(left).add(right); graph.get(right).add(left); + private static void validateDirectBinding(Map targetField, String reference) { + String column = getString(targetField, "dataObjectFieldName"); + try { + if (column == null || column.isBlank()) throw new IllegalArgumentException("missing physical column"); + MetricExpression.Node parsed = SqlMetricExpressionParser.parse(column, "ANSI_SQL"); + if (!(parsed instanceof MetricExpression.Field field) || field.parts().size() != 1 + || field.parts().get(0).quoted() || !column.equals(field.parts().get(0).text())) { + throw new IllegalArgumentException("expected one unquoted physical column"); } + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Field reference '" + reference + + "' was not exported with a supported direct physical binding; derived, qualified or quoted " + + "field bindings need separate conversion support: " + e.getMessage(), e); } - Map result = new HashMap<>(); - for (String name : names) { - if (result.containsKey(name)) continue; - int component = result.size(); - ArrayDeque pending = new ArrayDeque<>(); pending.add(name); - while (!pending.isEmpty()) { - String current = pending.removeFirst(); - if (result.putIfAbsent(current, component) == null) pending.addAll(graph.get(current)); - } + } + + private static Map exportedItem(List> items, + String name, String kind, String reference) { + List> matches = items.stream() + .filter(item -> name.equals(getString(item, "apiName"))).toList(); + if (matches.isEmpty()) { + throw new IllegalArgumentException("Declared " + kind + " in reference '" + reference + + "' was not exported as a direct Salesforce semantic " + kind + + "; calculated or omitted fields are unsupported in metric references"); + } + if (matches.size() > 1) { + throw new IllegalArgumentException("Ambiguous exported Salesforce " + kind + " for reference '" + + reference + "'; apiName values must be unique"); + } + return matches.get(0); + } + + private static boolean matches(Identifier reference, String declaration, boolean tableau) { + if (declaration == null) { + return false; } - return Map.copyOf(result); + return tableau ? reference.text().equals(declaration) + : normalize(reference).equals(normalizeDeclaration(declaration)); + } + + private static boolean equivalentDeclaration(String first, String second, boolean tableau) { + return second != null && (tableau ? first.equals(second) + : normalizeDeclaration(first).equals(normalizeDeclaration(second))); } - static String normalize(Identifier identifier) { + private static String normalize(Identifier identifier) { return identifier.quoted() ? identifier.text() : identifier.text().toUpperCase(Locale.ROOT); } - static String normalizeDeclaration(String name) { + private static String normalizeDeclaration(String name) { if (name.startsWith("\"") && name.endsWith("\"") && name.length() >= 2) { String text = name.substring(1, name.length() - 1); - if (text.isEmpty() || text.replace("\"\"", "").contains("\"")) { + String unescaped = text.replace("\"\"", ""); + if (text.isEmpty() || unescaped.contains("\"")) { throw new IllegalArgumentException("Invalid quoted declaration name '" + name + "'"); } return text.replace("\"\"", "\""); @@ -215,7 +289,7 @@ static String normalizeDeclaration(String name) { return name.toUpperCase(Locale.ROOT); } - static String bracket(String name) { + private static String bracket(String name) { if (name == null || name.isBlank() || name.indexOf('[') >= 0 || name.indexOf(']') >= 0 || name.chars().anyMatch(Character::isISOControl)) { throw new IllegalArgumentException("Exported apiName '" + name @@ -224,28 +298,6 @@ static String bracket(String name) { return "[" + name + "]"; } - private static Map> index(List values, Function name) { - Map> result = new HashMap<>(); - values.forEach(value -> result.computeIfAbsent(name.apply(value), ignored -> new ArrayList<>()).add(value)); - return result; - } - - private static final class NameIndex { - private final Map> exact = new HashMap<>(); - private final Map> sql = new HashMap<>(); - void add(String name, T value) { - if (name == null) return; - exact.computeIfAbsent(name, ignored -> new ArrayList<>()).add(value); - sql.computeIfAbsent(normalizeDeclaration(name), ignored -> new ArrayList<>()).add(value); - } - List find(Identifier identifier, boolean tableau) { - return (tableau ? exact : sql).getOrDefault(tableau ? identifier.text() : normalize(identifier), List.of()); - } - List declarations(String name, boolean tableau) { - return (tableau ? exact : sql).getOrDefault(tableau ? name : normalizeDeclaration(name), List.of()); - } - } - private static List> items(Map map, String key) { List values = getList(map, key); return values == null ? List.of() : streamMaps(values).toList(); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java index 05c553ae..5278abaf 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java @@ -58,25 +58,20 @@ public MetricMappingHandler(ConversionDirection direction, CustomExtensionHandle @Override public void execute(Map sourceData, Map outputData, Map mappings) { - execute(new ConversionContext(sourceData, outputData), mappings); - } - - @Override - public void execute(ConversionContext context, Map mappings) { logger.debug("Mapping metrics in {} direction", direction); if (direction == ConversionDirection.OSSIE_TO_SALESFORCE) { - mapOssieToSalesforce(context, mappings); + mapOssieToSalesforce(sourceData, outputData, mappings); } else { - mapSalesforceToOssie(context.sourceData(), context.outputData(), mappings); + mapSalesforceToOssie(sourceData, outputData, mappings); } } /** * Maps Ossie metrics to Salesforce semanticCalculatedMeasurements. */ - private void mapOssieToSalesforce(ConversionContext context, Map mappings) { - Map sourceData = context.sourceData(); - Map outputData = context.outputData(); + private void mapOssieToSalesforce( + Map sourceData, Map outputData, Map mappings) { + List ossieMetrics = getList(sourceData, METRICS); if (ossieMetrics == null) { return; @@ -100,7 +95,7 @@ private void mapOssieToSalesforce(ConversionContext context, Map List sfMetrics = getList(outputData, SEMANTIC_CALCULATED_MEASUREMENTS); if (sfMetrics != null) { - unwrapExpressions(ossieMetrics, sfMetrics, context); + unwrapExpressions(ossieMetrics, sfMetrics, sourceData, outputData); } else if (!ossieMetrics.isEmpty()) { throw new ConversionException("Metric '" + getString(asMap(ossieMetrics.get(0)), NAME) + "': metric mappings produced no calculated measurements"); @@ -147,28 +142,21 @@ private void mapSalesforceToOssie( * the OSI declarations and the actual emitted fields, including their types. */ private void unwrapExpressions(List ossieMetrics, List sfMetrics, - ConversionContext context) { + Map sourceData, Map outputData) { if (ossieMetrics.size() != sfMetrics.size()) { throw new ConversionException("Metric export count differs from declared metrics: " + streamMaps(ossieMetrics).map(metric -> getString(metric, NAME)).toList()); } - MetricFieldResolver resolver = new MetricFieldResolver(context.sourceData(), context.outputData(), context.fieldPlan()); - MetricCompilationPlan plan = new MetricCompilationPlan(context.sourceData(), resolver); + MetricFieldResolver resolver = new MetricFieldResolver(sourceData, outputData); for (int i = 0; i < ossieMetrics.size(); i++) { Map ossieMetric = asMap(ossieMetrics.get(i)); Map sfMetric = asMap(sfMetrics.get(i)); - customExtensionHandler.restoreSalesforceCustomExtension(sfMetric, ossieMetric); - ExpressionCompiler.Compiled translated = plan.compile(getString(ossieMetric, NAME)); - String nativeType = getString(sfMetric, DATA_TYPE); - if (nativeType != null && !Set.of("Number", "Currency", "Percentage").contains(nativeType)) { - throw new ConversionException("Metric '" + getString(ossieMetric, NAME) - + "': incompatible Salesforce dataType " + nativeType); - } + MetricExpressionTranslator.Result translated = + MetricExpressionTranslator.translate(ossieMetric, resolver); sfMetric.put(EXPRESSION, translated.expression()); - sfMetric.put(DATA_TYPE, nativeType == null ? "Number" : nativeType); + sfMetric.put(DATA_TYPE, translated.dataType()); sfMetric.put("syntax", "Tua"); sfMetric.put("aggregationType", "UserAgg"); - sfMetric.put("level", "AggregateFunction"); } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java index f1c30f9b..20c4949b 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/RelationshipMappingHandler.java @@ -68,18 +68,24 @@ public void execute(Map sourceData, Map outputDa private void mapOssieToSalesforce( Map sourceData, Map outputData, Map mappings) { - Map relationshipMappings = MappingUtils.filterMappingsByPrefix(mappings, RELATIONSHIPS); - // This handler owns relationships even when there are none. Leaving these entries - // behind lets the final generic handler copy raw Ossie relationships into the output. - relationshipMappings.keySet().forEach(mappings::remove); - List ossieRelationships = getList(sourceData, RELATIONSHIPS); if (ossieRelationships == null) { return; } - SalesforceModelValidator.validateRelationshipDeclarations(sourceData, outputData); + + // Validate and filter relationships - remove those with non-existent fields + List validRelationships = validateAndFilterRelationships(ossieRelationships, outputData); + if (validRelationships.isEmpty()) { + return; + } + + // Update sourceData with filtered relationships + sourceData.put(RELATIONSHIPS, validRelationships); + + Map relationshipMappings = MappingUtils.filterMappingsByPrefix(mappings, RELATIONSHIPS); Map mappedData = GenericMappingEngine.applyMappings(sourceData, relationshipMappings); + relationshipMappings.keySet().forEach(mappings::remove); outputData.putAll(mappedData); @@ -94,7 +100,6 @@ private void mapOssieToSalesforce( if (sfRelationships != null) { applyDefaults(sfRelationships); } - SalesforceModelValidator.validateRelationships(sourceData, outputData); } /** @@ -272,15 +277,110 @@ private void applyDefaults(List sfRelationships) { for (Object relObj : sfRelationships) { Map sfRel = asMap(relObj); - SalesforceModelValidator.validateRelationshipOptions(sfRel); - // Ossie defines `from` as the many side and `to` as the one side. - // An explicit native value remains an intentional round-trip override. - sfRel.putIfAbsent(CARDINALITY, "ManyToOne"); + sfRel.putIfAbsent(CARDINALITY, DEFAULT_CARDINALITY); sfRel.putIfAbsent(IS_ENABLED, true); sfRel.putIfAbsent(JOIN_TYPE, DEFAULT_JOIN_TYPE); } } + /** + * Validates and filters relationships, removing those that reference non-existent fields. (Calculated fields that are not supported) + * + * @param ossieRelationships List of Ossie relationships to validate + * @param outputData The output data containing semanticDataObjects with their fields + * @return Filtered list of valid relationships + */ + private List validateAndFilterRelationships(List ossieRelationships, Map outputData) { + List validRelationships = new ArrayList<>(); + List sfDataObjects = getList(outputData, SEMANTIC_DATA_OBJECTS); + + for (Object relObj : ossieRelationships) { + Map ossieRel = asMap(relObj); + String relName = getString(ossieRel, NAME); + String fromEntity = getString(ossieRel, FROM); + String toEntity = getString(ossieRel, TO); + + Map fromDataObject = findDataObjectByName(sfDataObjects, fromEntity); + Map toDataObject = findDataObjectByName(sfDataObjects, toEntity); + + if (fromDataObject == null || toDataObject == null) { + logger.debug("Removing relationship '{}' - entity not found", relName); + continue; + } + + List fromColumns = getList(ossieRel, FROM_COLUMNS); + List toColumns = getList(ossieRel, TO_COLUMNS); + + if (!validateColumns(fromColumns, fromDataObject, fromEntity, relName) || + !validateColumns(toColumns, toDataObject, toEntity, relName)) { + continue; + } + validRelationships.add(ossieRel); + } + return validRelationships; + } + + /** + * Validates that all columns exist in the given data object. + * + * @param columns List of column names to validate + * @param dataObject The data object containing the fields + * @param entityName The entity name (for logging) + * @param relName The relationship name (for logging) + * @return true if all columns exist, false otherwise + */ + private boolean validateColumns(List columns, Map dataObject, + String entityName, String relName) { + if (columns == null || columns.isEmpty()) { + return true; + } + + for (Object colObj : columns) { + String columnName = (String) colObj; + if (!fieldExistsInDataObject(dataObject, columnName)) { + logger.debug("Removing relationship '{}' - column '{}' not found in entity '{}'", + relName, columnName, entityName); + return false; + } + } + + return true; + } + + /** + * Finds a data object by its apiName. + */ + private Map findDataObjectByName(List dataObjects, String name) { + for (Object obj : dataObjects) { + Map dataObject = asMap(obj); + String apiName = getString(dataObject, API_NAME); + if (name.equals(apiName)) { + return dataObject; + } + } + return null; + } + + /** + * Checks if a field exists in a data object's semanticDimensions or semanticMeasurements. + */ + private boolean fieldExistsInDataObject(Map dataObject, String fieldName) { + // Check both semanticDimensions and semanticMeasurements + for (String fieldListKey : List.of(SEMANTIC_DIMENSIONS, SEMANTIC_MEASUREMENTS)) { + List fields = getList(dataObject, fieldListKey); + if (fields != null) { + for (Object fieldObj : fields) { + Map field = asMap(fieldObj); + String apiName = getString(field, API_NAME); + if (fieldName.equals(apiName)) { + return true; + } + } + } + } + return false; + } + /** * Checks if a relationship has unsupported field types (Formula or SemanticField). * diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java deleted file mode 100644 index 0be0cc68..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceBindings.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.util.DataStructureUtils.*; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.ossie.exception.ConversionException; -import org.apache.ossie.exception.InvalidInputException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** Immutable environment bindings. Business expressions and semantic names cannot be overridden. */ -public final class SalesforceBindings { - private record DatasetBinding(String objectName, String objectType, Map fields) {} - private record ModelBinding(String dataspace, Map datasets) {} - private final Map models; - - private SalesforceBindings(Map models) { this.models = Map.copyOf(models); } - - public static SalesforceBindings none() { return new SalesforceBindings(Map.of()); } - - /** Loads JSON or YAML. Unknown properties and duplicate mapping keys are errors. */ - public static SalesforceBindings fromPath(Path path) { - try { return fromString(Files.readString(path)); } - catch (IOException e) { throw new InvalidInputException("Cannot read Salesforce bindings: " + path, e); } - } - - public static SalesforceBindings fromString(String content) { - try { - YAMLFactory factory = new YAMLFactory(); - factory.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); - Map root = new ObjectMapper(factory).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) - .readValue(content, new TypeReference<>() {}); - if (root == null) throw new IllegalArgumentException("bindings document is empty"); - keys(root, Set.of("models"), "bindings"); - Map entries = object(root.get("models"), "bindings.models"); - Map result = new LinkedHashMap<>(); - for (var entry : entries.entrySet()) { - String context = "Bindings for model '" + entry.getKey() + "'"; - Map model = object(entry.getValue(), context); - keys(model, Set.of("dataspace", "datasets"), context); - Map datasets = new LinkedHashMap<>(); - for (var dataset : optionalObject(model, "datasets", context).entrySet()) { - String location = context + ", dataset '" + dataset.getKey() + "'"; - Map definition = object(dataset.getValue(), location); - keys(definition, Set.of("dataObjectName", "dataObjectType", "fields"), location); - Map fields = new LinkedHashMap<>(); - for (var field : optionalObject(definition, "fields", location).entrySet()) { - fields.put(field.getKey(), string(field.getValue(), location + ", field '" + field.getKey() + "'")); - } - datasets.put(dataset.getKey(), new DatasetBinding(optionalString(definition, "dataObjectName", location), - optionalString(definition, "dataObjectType", location), Map.copyOf(fields))); - } - result.put(entry.getKey(), new ModelBinding(optionalString(model, "dataspace", context), Map.copyOf(datasets))); - } - return new SalesforceBindings(result); - } catch (IOException | IllegalArgumentException e) { - throw new InvalidInputException("Invalid Salesforce bindings: " + e.getMessage(), e); - } - } - - boolean isEmpty() { return models.isEmpty(); } - - void validateModels(Set names) { - for (String name : models.keySet()) { - if (!names.contains(name)) throw new ConversionException("Bindings reference unknown model '" + name + "'"); - } - } - - /** Apply only after expressions are bound in their original source scope. */ - void apply(Map source, Map target) { - String modelName = getString(source, "name"); - ModelBinding model = models.get(modelName); - if (model == null) return; - if (model.dataspace != null) target.put("dataspace", model.dataspace); - Map>> sourceDatasets = index(items(source, "datasets"), "name"); - Map>> targetDatasets = index(items(target, "semanticDataObjects"), "apiName"); - for (var entry : model.datasets.entrySet()) { - String datasetName = entry.getKey(); - String context = "Bindings for model '" + modelName + "', dataset '" + datasetName + "'"; - Map sourceDataset = unique(sourceDatasets, datasetName, context); - Map targetDataset = unique(targetDatasets, datasetName, context); - DatasetBinding binding = entry.getValue(); - if (binding.objectName != null) targetDataset.put("dataObjectName", binding.objectName); - if (binding.objectType != null) targetDataset.put("dataObjectType", binding.objectType); - Map>> sourceFields = index(items(sourceDataset, "fields"), "name"); - List> directFields = new java.util.ArrayList<>(items(targetDataset, "semanticDimensions")); - directFields.addAll(items(targetDataset, "semanticMeasurements")); - Map>> targetFields = index(directFields, "apiName"); - for (var field : binding.fields.entrySet()) { - unique(sourceFields, field.getKey(), context + ", field '" + field.getKey() + "'"); - Map targetField = unique(targetFields, field.getKey(), - context + ", field '" + field.getKey() + "' (only direct physical fields can be rebound)"); - targetField.put("dataObjectFieldName", field.getValue()); - } - } - } - - private static Map>> index(List> objects, String key) { - Map>> result = new LinkedHashMap<>(); - for (Map object : objects) { - result.computeIfAbsent(getString(object, key), ignored -> new java.util.ArrayList<>()).add(object); - } - return result; - } - - private static Map unique(Map>> objects, String name, String context) { - List> matches = objects.getOrDefault(name, List.of()); - if (matches.size() != 1) throw new ConversionException(context + ": expected one declared/exported identity, found " + matches.size()); - return matches.get(0); - } - - private static void keys(Map value, Set allowed, String context) { - for (String key : value.keySet()) { - if (!allowed.contains(key)) throw new IllegalArgumentException(context + ": unknown property '" + key + "'"); - } - } - - @SuppressWarnings("unchecked") - private static Map object(Object value, String context) { - if (!(value instanceof Map map)) throw new IllegalArgumentException(context + " must be an object"); - if (map.keySet().stream().anyMatch(key -> !(key instanceof String) || ((String) key).isBlank())) { - throw new IllegalArgumentException(context + " requires nonempty string keys"); - } - return (Map) map; - } - - private static Map optionalObject(Map value, String key, String context) { - return value.containsKey(key) ? object(value.get(key), context + "." + key) : Map.of(); - } - - private static String string(Object value, String context) { - if (!(value instanceof String text) || text.isBlank() || text.chars().anyMatch(Character::isISOControl)) { - throw new IllegalArgumentException(context + " must be a nonempty string without control characters"); - } - return text; - } - - private static String optionalString(Map value, String key, String context) { - return value.containsKey(key) ? string(value.get(key), context + "." + key) : null; - } - - private static List> items(Map value, String key) { - List list = getList(value, key); - return list == null ? List.of() : streamMaps(list).toList(); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java deleted file mode 100644 index a96488fc..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceModelValidator.java +++ /dev/null @@ -1,567 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.converter.ConverterConstants.*; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.ArrayDeque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import org.apache.ossie.exception.ConversionException; - -/** - * Checks references and preservation after all handlers and native extensions have run. - * JSON schema validation remains separate: a structurally valid model can still contain - * duplicate identities, dangling references, omitted entities, or disconnected calculations. - * This validates declared metadata, not actual uniqueness or data in a Salesforce catalog. - */ -public final class SalesforceModelValidator { - private static final ObjectMapper JSON = new ObjectMapper(); - private static final Set CARDINALITIES = Set.of( - "OneToOne", "OneToMany", "ManyToOne", "ManyToMany", "Unspecified"); - private static final Set JOIN_TYPES = Set.of("Auto", "Inner", "Left", "Right", "Full"); - private static final List DIRECT_FIELDS = List.of(SEMANTIC_DIMENSIONS, SEMANTIC_MEASUREMENTS); - private static final List CALCULATED_FIELDS = - List.of(SEMANTIC_CALCULATED_DIMENSIONS, SEMANTIC_CALCULATED_MEASUREMENTS); - - /** Validates a model whose source fields are all direct bindings. */ - public void validate(Map sourceModel, Map targetModel) { - validate(sourceModel, targetModel, null); - } - - /** Uses the existing field plan to verify coverage without compiling expressions again. */ - public void validate(Map sourceModel, Map targetModel, - FieldExpressionPlan fieldPlan) { - String modelName = text(sourceModel, NAME, "Ossie model"); - if (!modelName.equals(text(targetModel, API_NAME, "Salesforce model"))) { - fail("Model '" + modelName + "' changed identity during conversion"); - } - Map> sources = index(items(sourceModel, DATASETS), NAME, "dataset", true); - Map> targets = index(items(targetModel, SEMANTIC_DATA_OBJECTS), API_NAME, - "exported dataset", false); - Map> calculations = calculationIndex(targetModel); - Set compiledCalculations = new HashSet<>(); - Map>> targetFields = new LinkedHashMap<>(); - for (var entry : targets.entrySet()) { - String scope = "dataset '" + entry.getKey() + "'"; - text(entry.getValue(), "dataObjectName", scope); - Map> fields = directFields(entry.getValue()); - for (var field : fields.entrySet()) { - text(field.getValue(), DATA_OBJECT_FIELD_NAME, scope + " field '" + field.getKey() + "'"); - } - targetFields.put(entry.getKey(), fields); - } - for (var entry : sources.entrySet()) { - String dataset = entry.getKey(); - require(targets, dataset, "Declared dataset '" + dataset + "' was not exported"); - Map> fields = sourceFields(entry.getValue()); - validateKeys(entry.getValue(), fields, dataset); - for (String field : fields.keySet()) { - if (targetFields.get(dataset).containsKey(field)) { - continue; - } - String calculated = fieldPlan == null ? null : fieldPlan.calculatedApiName(dataset, field); - if (calculated == null || !calculations.containsKey(calculated)) { - fail("Declared field '" + dataset + "." + field + "' was not exported"); - } - compiledCalculations.add(calculated); - } - } - Map> sourceMetrics = index(items(sourceModel, METRICS), NAME, "metric", true); - Map> targetMetrics = index(items(targetModel, SEMANTIC_CALCULATED_MEASUREMENTS), - API_NAME, "exported metric", false); - for (String metric : sourceMetrics.keySet()) { - require(targetMetrics, metric, "Declared metric '" + metric + "' was not exported"); - compiledCalculations.add(metric); - } - validateNativeArrayCoverage(sourceModel, targetModel); - validateRelationshipDeclarations(sourceModel, targetModel); - validateRelationships(sourceModel, targetModel); - validateCalculations(calculations, targetFields, targetModel, compiledCalculations); - } - - private static void validateNativeArrayCoverage(Map source, Map target) { - for (Map extension : items(source, CUSTOM_EXTENSIONS)) { - if (!VENDOR_NAME_VALUE.equals(extension.get(VENDOR_NAME))) continue; - Map nativeModel; - try { - nativeModel = JSON.readValue(text(extension, DATA, "Salesforce extension"), new TypeReference<>() {}); - } catch (java.io.IOException e) { - throw new ConversionException("Model Salesforce extension contains invalid JSON", e); - } - if (nativeModel == null) fail("Model Salesforce extension must contain a JSON object"); - for (String array : List.of(SEMANTIC_DATA_OBJECTS, SEMANTIC_RELATIONSHIPS, - SEMANTIC_CALCULATED_DIMENSIONS, SEMANTIC_CALCULATED_MEASUREMENTS)) { - Map> expected = index(items(nativeModel, array), API_NAME, - "native extension " + array, false); - Map> actual = index(items(target, array), API_NAME, array, false); - for (String name : expected.keySet()) { - require(actual, name, "Native extension " + array + " entity '" + name - + "' was not exported; merge it into the core model instead of silently replacing its array"); - } - } - } - } - - /** Checks every declared edge before mapping; no field or edge may be silently dropped. */ - static void validateRelationshipDeclarations(Map source, Map target) { - Map> sources = index(items(source, DATASETS), NAME, "dataset", true); - Map> targets = index(items(target, SEMANTIC_DATA_OBJECTS), API_NAME, - "exported dataset", false); - Map> relationships = index(items(source, RELATIONSHIPS), NAME, - "relationship", true); - for (var entry : relationships.entrySet()) { - String scope = "Relationship '" + entry.getKey() + "'"; - Map relation = entry.getValue(); - String from = text(relation, FROM, scope); - String to = text(relation, TO, scope); - Map fromDataset = require(sources, from, scope + " has unknown from dataset '" + from + "'"); - Map toDataset = require(sources, to, scope + " has unknown to dataset '" + to + "'"); - Map fromTarget = require(targets, from, scope + " has unexported dataset '" + from + "'"); - Map toTarget = require(targets, to, scope + " has unexported dataset '" + to + "'"); - List fromColumns = names(relation.get(FROM_COLUMNS), scope + " from_columns"); - List toColumns = names(relation.get(TO_COLUMNS), scope + " to_columns"); - if (fromColumns.size() != toColumns.size()) { - fail(scope + " must have the same number of from_columns and to_columns"); - } - checkColumns(scope, from, fromColumns, fromDataset, fromTarget); - checkColumns(scope, to, toColumns, toDataset, toTarget); - validateKeys(toDataset, sourceFields(toDataset), to); - validateKeys(fromDataset, sourceFields(fromDataset), from); - } - } - - /** Validates final native edges, including relationships restored from native extensions. */ - static void validateRelationships(Map source, Map target) { - Map> sourceDatasets = index(items(source, DATASETS), NAME, "dataset", true); - Map> datasets = index(items(target, SEMANTIC_DATA_OBJECTS), API_NAME, - "exported dataset", false); - Map> relationships = index(items(target, SEMANTIC_RELATIONSHIPS), API_NAME, - "exported relationship", false); - for (var entry : relationships.entrySet()) { - String scope = "Relationship '" + entry.getKey() + "'"; - Map relation = entry.getValue(); - String left = text(relation, LEFT_SEMANTIC_DEFINITION_API_NAME, scope); - String right = text(relation, RIGHT_SEMANTIC_DEFINITION_API_NAME, scope); - Map leftDataset = require(datasets, left, scope + " has unknown endpoint '" + left + "'"); - Map rightDataset = require(datasets, right, scope + " has unknown endpoint '" + right + "'"); - List> criteria = items(relation, CRITERIA); - if (criteria.isEmpty()) fail(scope + " requires nonempty criteria"); - for (Map criterion : criteria) { - validateCriterion(scope, criterion, LEFT_FIELD_TYPE, LEFT_SEMANTIC_FIELD_API_NAME, left, leftDataset); - validateCriterion(scope, criterion, RIGHT_FIELD_TYPE, RIGHT_SEMANTIC_FIELD_API_NAME, right, rightDataset); - } - validateRelationshipOptions(relation); - } - for (Map relation : items(source, RELATIONSHIPS)) { - String name = text(relation, NAME, "Ossie relationship"); - Map exported = require(relationships, name, - "Declared relationship '" + name + "' was not exported"); - String scope = "Relationship '" + name + "'"; - if (!text(relation, FROM, scope).equals(exported.get(LEFT_SEMANTIC_DEFINITION_API_NAME)) - || !text(relation, TO, scope).equals(exported.get(RIGHT_SEMANTIC_DEFINITION_API_NAME))) { - fail(scope + " changed endpoints during conversion"); - } - List from = names(relation.get(FROM_COLUMNS), scope + " from_columns"); - List to = names(relation.get(TO_COLUMNS), scope + " to_columns"); - List> criteria = items(exported, CRITERIA); - if (from.size() != to.size() || criteria.size() != from.size()) fail(scope + " changed composite key arity"); - for (int i = 0; i < from.size(); i++) { - if (!from.get(i).equals(criteria.get(i).get(LEFT_SEMANTIC_FIELD_API_NAME)) - || !to.get(i).equals(criteria.get(i).get(RIGHT_SEMANTIC_FIELD_API_NAME))) { - fail(scope + " changed join key correspondence during conversion"); - } - } - // Native round-trip metadata can explicitly reverse or relax core cardinality. - // Check keys on the side(s) that the final native edge actually declares unique. - String cardinality = (String) exported.get(CARDINALITY); - if ("ManyToOne".equals(cardinality) || "OneToOne".equals(cardinality)) { - validateUniqueSide(scope, "to_columns", to, sourceDatasets.get(relation.get(TO))); - } - if ("OneToMany".equals(cardinality) || "OneToOne".equals(cardinality)) { - validateUniqueSide(scope, "from_columns", from, sourceDatasets.get(relation.get(FROM))); - } - } - } - - static void validateRelationshipOptions(Map relation) { - String scope = "Relationship '" + relation.get(API_NAME) + "'"; - if (relation.containsKey(CARDINALITY)) { - String cardinality = text(relation, CARDINALITY, scope); - if (!CARDINALITIES.contains(cardinality)) fail(scope + " has invalid cardinality '" + cardinality + "'"); - } - if (relation.containsKey(JOIN_TYPE)) { - String joinType = text(relation, JOIN_TYPE, scope); - if (!JOIN_TYPES.contains(joinType)) fail(scope + " has invalid joinType '" + joinType + "'"); - } - if (relation.containsKey(IS_ENABLED) && !(relation.get(IS_ENABLED) instanceof Boolean)) { - fail(scope + " requires a Boolean isEnabled when supplied"); - } - } - - private static void validateUniqueSide(String scope, String side, List columns, Map dataset) { - if (dataset == null) fail(scope + " references an undeclared dataset"); - String name = (String) dataset.get(NAME); - List> keys = validateKeys(dataset, sourceFields(dataset), name); - if (!keys.isEmpty() && keys.stream().noneMatch(columns::containsAll)) { - fail(scope + " " + side + " do not include a declared primary_key or unique_key of '" + name + "'"); - } - } - - private static void validateCriterion(String scope, Map criterion, String typeKey, - String fieldKey, String dataset, Map target) { - Object type = criterion.get(typeKey); - if (type != null && !FIELD_TYPE_TABLE_FIELD.equals(type)) { - fail(scope + " uses unsupported calculated join key type '" + type + "'; only direct table fields are supported"); - } - String field = text(criterion, fieldKey, scope + " criterion"); - require(directFields(target), field, scope + " references missing direct field '" + dataset + "." + field + "'"); - } - - private static void checkColumns(String scope, String dataset, List columns, - Map source, Map target) { - Map> declared = sourceFields(source); - Map> exported = directFields(target); - for (String column : columns) { - require(declared, column, scope + " references undeclared field '" + dataset + "." + column + "'"); - require(exported, column, scope + " field '" + dataset + "." + column - + "' was not exported as a direct field; calculated join keys are unsupported"); - } - } - - private static List> validateKeys(Map dataset, - Map> fields, String name) { - List> keys = new ArrayList<>(); - if (dataset.containsKey("primary_key")) { - keys.add(new LinkedHashSet<>(names(dataset.get("primary_key"), "Dataset '" + name + "' primary_key"))); - } - Object unique = dataset.get("unique_keys"); - if (unique != null) { - if (!(unique instanceof List)) fail("Dataset '" + name + "' unique_keys must be an array"); - for (Object key : (List) unique) { - keys.add(new LinkedHashSet<>(names(key, "Dataset '" + name + "' unique_key"))); - } - } - for (Set key : keys) { - for (String field : key) require(fields, field, "Dataset '" + name + "' key references unknown field '" + field + "'"); - } - // The native schema has no primary/unique-key constraint. primaryNameField is - // a display identifier, and keyQualifierName is not a uniqueness declaration. - return keys; - } - - private static void validateCalculations(Map> calculations, - Map>> fields, Map target, - Set compiledCalculations) { - Map> dependencies = new LinkedHashMap<>(); - Map> datasets = new LinkedHashMap<>(); - for (var entry : calculations.entrySet()) { - String scope = "Calculated field '" + entry.getKey() + "'"; - if (!"Tua".equals(entry.getValue().get("syntax"))) fail(scope + " requires syntax 'Tua'"); - String expression = text(entry.getValue(), EXPRESSION, scope); - Set refs = new LinkedHashSet<>(); - Set sources = new LinkedHashSet<>(); - for (List reference : references(expression, scope)) { - if (reference.size() == 2) { - String dataset = reference.get(0); - Map> table = require(fields, dataset, - scope + " references unknown dataset '" + dataset + "'"); - require(table, reference.get(1), scope + " references missing field '" - + dataset + "." + reference.get(1) + "'"); - sources.add(dataset); - } else { - String name = reference.get(0); - require(calculations, name, scope + " references unknown calculated field '" + name + "'"); - refs.add(name); - } - } - for (Map dependency : items(entry.getValue(), DEPENDENCIES)) { - String definition = text(dependency, DEPENDENT_DEFINITION_API_NAME, scope + " dependency"); - Object field = dependency.get("dependentFieldApiName"); - if (field != null) { - Map> table = require(fields, definition, - scope + " dependency references unknown dataset '" + definition + "'"); - String name = text(dependency, "dependentFieldApiName", scope + " dependency"); - require(table, name, scope + " dependency references missing field '" + definition + "." + name + "'"); - } else if (!fields.containsKey(definition) && !calculations.containsKey(definition)) { - fail(scope + " dependency references unknown definition '" + definition + "'"); - } - } - dependencies.put(entry.getKey(), refs); - datasets.put(entry.getKey(), sources); - } - List order = collectDatasets(dependencies, datasets); - validateNativeCalculations(order, calculations, fields, datasets, target, compiledCalculations); - Map> graph = new LinkedHashMap<>(); - fields.keySet().forEach(name -> graph.put(name, new LinkedHashSet<>())); - for (Map relation : items(target, SEMANTIC_RELATIONSHIPS)) { - // Optional native metadata may be absent. Only explicitly enabled edges - // establish connectivity; absence is not proof of the target's default. - if (!Boolean.TRUE.equals(relation.get(IS_ENABLED))) continue; - String left = (String) relation.get(LEFT_SEMANTIC_DEFINITION_API_NAME); - String right = (String) relation.get(RIGHT_SEMANTIC_DEFINITION_API_NAME); - graph.get(left).add(right); - graph.get(right).add(left); - } - for (var entry : datasets.entrySet()) { - if (entry.getValue().size() < 2) continue; - Set reached = new HashSet<>(); - List pending = new ArrayList<>(List.of(entry.getValue().iterator().next())); - for (int i = 0; i < pending.size(); i++) { - String dataset = pending.get(i); - if (reached.add(dataset)) pending.addAll(graph.get(dataset)); - } - if (!reached.containsAll(entry.getValue())) { - fail("Calculated field '" + entry.getKey() + "' references datasets disconnected by enabled relationships"); - } - } - } - - private static List collectDatasets(Map> dependencies, Map> datasets) { - Map remaining = new LinkedHashMap<>(); - Map> consumers = new HashMap<>(); - ArrayDeque ready = new ArrayDeque<>(); - for (var entry : dependencies.entrySet()) { - remaining.put(entry.getKey(), entry.getValue().size()); - if (entry.getValue().isEmpty()) ready.add(entry.getKey()); - for (String dependency : entry.getValue()) { - consumers.computeIfAbsent(dependency, ignored -> new ArrayList<>()).add(entry.getKey()); - } - } - int processed = 0; - List order = new ArrayList<>(); - while (!ready.isEmpty()) { - String name = ready.removeFirst(); - order.add(name); - processed++; - for (String consumer : consumers.getOrDefault(name, List.of())) { - datasets.get(consumer).addAll(datasets.get(name)); - if (remaining.compute(consumer, (ignored, count) -> count - 1) == 0) ready.add(consumer); - } - } - if (processed != dependencies.size()) { - String cyclic = remaining.entrySet().stream().filter(entry -> entry.getValue() > 0) - .map(Map.Entry::getKey).findFirst().orElseThrow(); - fail("Cyclic calculated field reference involving '" + cyclic + "'"); - } - return order; - } - - private static void validateNativeCalculations(List order, - Map> calculations, - Map>> fields, Map> datasets, - Map target, Set alreadyCompiled) { - Set measurements = index(items(target, SEMANTIC_CALCULATED_MEASUREMENTS), API_NAME, - "calculated measurement", false).keySet(); - Map verified = new HashMap<>(); - for (String name : order) { - if (alreadyCompiled.contains(name)) continue; - Map calculation = calculations.get(name); - String scope = "Native calculated field '" + name + "'"; - try { - ExpressionCompiler.Compiled compiled = ExpressionCompiler.compile( - ExpressionCompiler.parse(text(calculation, EXPRESSION, scope), DIALECT_TABLEAU), reference -> { - List parts = reference.parts(); - if (parts.size() == 2) { - String dataset = parts.get(0).text(); - String field = parts.get(1).text(); - Map> table = require(fields, dataset, - scope + " references unknown dataset '" + dataset + "'"); - Map item = require(table, field, scope + " references missing field '" + field + "'"); - return new ExpressionCompiler.Binding("[" + dataset + "].[" + field + "]", - nativeDatatype(item, scope + " reference '" + dataset + "." + field + "'"), - dataset, ExpressionCompiler.Level.ROW); - } - if (parts.size() != 1) throw new IllegalArgumentException("reference must be dataset.field or calculated field"); - String dependency = parts.get(0).text(); - Map item = require(calculations, dependency, - scope + " references unknown calculated field '" + dependency + "'"); - ExpressionCompiler.Compiled checked = verified.get(dependency); - String datatype = checked == null ? nativeDatatype(item, scope + " reference '" + dependency + "'") - : checked.datatype(); - ExpressionCompiler.Level level = checked == null - ? measurements.contains(dependency) ? ExpressionCompiler.Level.AGGREGATE : ExpressionCompiler.Level.ROW - : checked.level(); - return new ExpressionCompiler.Binding("[" + dependency + "]", datatype, datasets.get(dependency), level); - }); - String type = calculation.get(DATA_TYPE) instanceof String value ? value : null; - String datatype = compiled.datatype() == null ? SalesforceDataTypeMapper.toOssie(type) : compiled.datatype(); - if (SalesforceDataTypeMapper.toSalesforce(datatype) == null) { - throw new IllegalArgumentException("expression needs a supported native dataType to determine its result type"); - } - boolean measurement = measurements.contains(name); - if (measurement && !Set.of("Integer", "Decimal", "Float").contains(datatype)) { - throw new IllegalArgumentException("calculated measurements must return a number"); - } - String expectedLevel = calculation.containsKey("level") ? text(calculation, "level", scope) - : measurement ? "AggregateFunction" : "Row"; - if (!Set.of("Row", "AggregateFunction").contains(expectedLevel)) { - throw new IllegalArgumentException("unsupported native level '" + expectedLevel + "'"); - } - if (compiled.level() == ExpressionCompiler.Level.AGGREGATE && expectedLevel.equals("Row") - || compiled.level() == ExpressionCompiler.Level.ROW && expectedLevel.equals("AggregateFunction")) { - throw new IllegalArgumentException("expression aggregation conflicts with native level '" + expectedLevel + "'"); - } - if (type != null && !SalesforceDataTypeMapper.areCompatible(datatype, type)) { - throw new IllegalArgumentException("expression datatype '" + datatype - + "' conflicts with native dataType '" + type + "'"); - } - verified.put(name, new ExpressionCompiler.Compiled(compiled.expression(), datatype, - compiled.level(), compiled.datasets())); - } catch (IllegalArgumentException e) { - throw new ConversionException(scope + ": " + e.getMessage(), e); - } - } - } - - private static String nativeDatatype(Map item, String scope) { - String type = SalesforceDataTypeMapper.toOssie(item.get(DATA_TYPE) instanceof String value ? value : null); - if (SalesforceDataTypeMapper.toSalesforce(type) == null) fail(scope + " needs a supported dataType"); - return type; - } - - // A reference scanner, not a second formula parser. The compiler owns grammar and - // type validation; this checks final names after native extensions have been restored. - private static List> references(String expression, String scope) { - List> references = new ArrayList<>(); - for (int i = 0; i < expression.length();) { - char c = expression.charAt(i); - if (c == '\'' || c == '"') { - char quote = c; - i++; - boolean closed = false; - while (i < expression.length()) { - char next = expression.charAt(i++); - if (next == '\\' && i < expression.length()) { i++; continue; } - if (next == quote) { - if (i < expression.length() && expression.charAt(i) == quote) { i++; continue; } - closed = true; - break; - } - } - if (!closed) fail(scope + " has an unterminated string literal"); - } else if (c == '[') { - List parts = new ArrayList<>(); - do { - int end = expression.indexOf(']', i + 1); - if (end < 0 || end == i + 1) fail(scope + " has an invalid bracket reference"); - parts.add(expression.substring(i + 1, end)); - i = end + 1; - while (i < expression.length() && Character.isWhitespace(expression.charAt(i))) i++; - if (i >= expression.length() || expression.charAt(i) != '.') break; - i++; - while (i < expression.length() && Character.isWhitespace(expression.charAt(i))) i++; - if (i >= expression.length() || expression.charAt(i) != '[') fail(scope + " has an invalid qualified reference"); - } while (true); - if (parts.size() > 2) fail(scope + " references more than dataset.field"); - references.add(parts); - } else { - i++; - } - } - return references; - } - - private static Map> calculationIndex(Map model) { - List> calculations = new ArrayList<>(); - for (String key : CALCULATED_FIELDS) calculations.addAll(items(model, key)); - return index(calculations, API_NAME, "model-level calculated field", false); - } - - private static Map> sourceFields(Map dataset) { - return index(items(dataset, FIELDS), NAME, "field in dataset '" + dataset.get(NAME) + "'", true); - } - - private static Map> directFields(Map dataset) { - List> fields = new ArrayList<>(); - for (String key : DIRECT_FIELDS) fields.addAll(items(dataset, key)); - return index(fields, API_NAME, "field in exported dataset '" + dataset.get(API_NAME) + "'", false); - } - - private static Map> index(List> values, - String key, String kind, boolean normalize) { - Map> result = new LinkedHashMap<>(); - Set identities = new HashSet<>(); - for (Map value : values) { - String name = text(value, key, kind); - String identity = normalize ? normalize(name) : name; - if (!identities.add(identity)) fail("Duplicate " + kind + " identity '" + name + "'"); - result.put(name, value); - } - return result; - } - - private static String normalize(String name) { - return name.length() >= 2 && name.startsWith("\"") && name.endsWith("\"") - ? name.substring(1, name.length() - 1).replace("\"\"", "\"") : name.toUpperCase(Locale.ROOT); - } - - private static List names(Object value, String scope) { - if (!(value instanceof List values) || values.isEmpty()) fail(scope + " must be a nonempty array"); - List result = new ArrayList<>(); - Set unique = new HashSet<>(); - for (Object item : (List) value) { - if (!(item instanceof String name) || name.isBlank()) fail(scope + " contains a blank or non-string field name"); - String name = (String) item; - if (!unique.add(name)) fail(scope + " contains duplicate field '" + name + "'"); - result.add(name); - } - return result; - } - - @SuppressWarnings("unchecked") - private static List> items(Map object, String key) { - Object value = object.get(key); - if (value == null) return List.of(); - if (!(value instanceof List)) fail("Property '" + key + "' must be an array"); - List> result = new ArrayList<>(); - for (Object item : (List) value) { - if (!(item instanceof Map)) fail("Property '" + key + "' must contain objects"); - result.add((Map) item); - } - return result; - } - - private static String text(Map object, String key, String scope) { - Object value = object.get(key); - if (!(value instanceof String text) || text.isBlank()) fail(scope + " requires a nonempty '" + key + "'"); - return (String) value; - } - - private static T require(Map values, String key, String error) { - T value = values.get(key); - if (value == null) fail(error); - return value; - } - - private static void fail(String message) { - throw new ConversionException(message); - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java similarity index 90% rename from converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java index 73838d16..ef9e7583 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlExpressionParser.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java @@ -19,7 +19,7 @@ package org.apache.ossie.converter; -import static org.apache.ossie.converter.ExpressionAst.*; +import static org.apache.ossie.converter.MetricExpression.*; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -32,17 +32,18 @@ import net.sf.jsqlparser.statement.select.AllColumns; /** Adapts a completely consumed JSqlParser expression into the explicitly supported compiler AST. */ -final class SqlExpressionParser { +final class SqlMetricExpressionParser { private final String dialect; private int depth; - private SqlExpressionParser(String dialect) { this.dialect = dialect; } + private SqlMetricExpressionParser(String dialect) { this.dialect = dialect; } static Node parse(String text, String dialect) { + TuaMetricExpressionParser.tokenize(text, dialect); try { Expression expression = CCJSqlParserUtil.parseCondExpression(text, false, parser -> parser.withSquareBracketQuotation(dialect.equals("ANSI_SQL"))); if (expression == null) throw new IllegalArgumentException("could not parse a complete SQL expression"); - return new SqlExpressionParser(dialect).adapt(expression); + return new SqlMetricExpressionParser(dialect).adapt(expression); } catch (net.sf.jsqlparser.JSQLParserException e) { throw new IllegalArgumentException(dialect + " expression has unsupported or unexpected token: " + e.getMessage(), e); @@ -62,7 +63,7 @@ private Node adaptNode(Expression expression) { return adapt(list.get(0)); } if (expression instanceof LongValue || expression instanceof DoubleValue) { - return new Literal(ExpressionTokens.number(expression.toString())); + return new Literal(TuaMetricExpressionParser.number(expression.toString())); } if (expression instanceof NullValue) return new Literal(null); if (expression instanceof BooleanValue value) return new Literal(value.getValue()); @@ -120,8 +121,7 @@ private Node function(Function function) { if (name == null || !name.matches("[A-Za-z_][A-Za-z_0-9]*")) { throw new IllegalArgumentException("quoted or qualified function names are unsupported"); } - boolean positionSyntax = name.equalsIgnoreCase("POSITION") && function.getNamedParameters() != null; - if (function.isUnique() || function.isEscaped() || function.getNamedParameters() != null && !positionSyntax + if (function.isUnique() || function.isEscaped() || function.getNamedParameters() != null || function.getAttribute() != null || function.getKeep() != null || function.getNullHandling() != null || function.isIgnoreNullsOutside() || function.isIgnoreNulls() || function.getLimit() != null @@ -134,14 +134,6 @@ private Node function(Function function) { throw new IllegalArgumentException("explicit ALL function modifier is outside the supported SQL subset"); } List arguments = new ArrayList<>(); - if (positionSyntax) { - var named = function.getNamedParameters(); - if (named.size() != 2 || named.getNames().size() != 2 - || !"IN".equalsIgnoreCase(named.getNames().get(1))) { - throw new IllegalArgumentException("unsupported POSITION argument syntax"); - } - for (Expression argument : named) arguments.add(adapt(argument)); - } if (function.getParameters() != null) { for (Expression argument : function.getParameters()) arguments.add(adapt(argument)); } @@ -177,7 +169,7 @@ private Node column(Column column) { } parts.add(new MetricFieldResolver.Identifier(part, quoted)); } - return new Field(new ExpressionCompiler.Reference(parts, bracket)); + return new Field(parts, bracket); } private IllegalArgumentException unsupported(Expression expression) { diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java deleted file mode 100644 index a50efcfa..00000000 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionEmitter.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.apache.ossie.converter.ExpressionAst.*; -import java.math.BigDecimal; -import java.util.List; - -/** Emits only checked AST nodes into the target grammar, with bounded expansion. */ -final class TuaExpressionEmitter { - private static final int MAX_OUTPUT = 131072; - private final StringBuilder output = new StringBuilder(); - String emit(Typed expression) { append(expression); return output.toString(); } - private void text(String text) { - if ((long) output.length() + text.length() > MAX_OUTPUT) { - throw new IllegalArgumentException("translated expression exceeds 131072 characters"); - } - output.append(text); - } - private void append(Typed value) { - Node node = value.node(); List children = value.children(); - if (node instanceof Literal literal) { - Object content = literal.value(); - text(content == null ? "NULL" : content instanceof String string ? "'" + string.replace("'", "''") + "'" - : content instanceof Boolean bool ? bool ? "TRUE" : "FALSE" : ((BigDecimal) content).toPlainString()); - } else if (node instanceof Field) { - text(value.binding().expression()); - } else if (node instanceof Unary unary) { - switch (unary.operator()) { - case "+" -> append(children.get(0)); - case "-" -> { text("(-"); append(children.get(0)); text(")"); } - case "NOT" -> { text("(NOT "); append(children.get(0)); text(")"); } - case "ISNULL" -> { text("ISNULL("); append(children.get(0)); text(")"); } - default -> throw new IllegalStateException("unvalidated unary operator"); - } - } else if (node instanceof Binary binary) { - text("("); append(children.get(0)); text(" " + binary.operator() + " "); append(children.get(1)); text(")"); - } else if (node instanceof Conditional) { - text("(IF "); - for (int i = 0; i < children.size() - 1; i += 2) { - if (i > 0) text(" ELSEIF "); - append(children.get(i)); text(" THEN "); append(children.get(i + 1)); - } - text(" ELSE "); append(children.get(children.size() - 1)); text(" END)"); - } else { - Call call = (Call) node; - ExpressionFunctionRegistry.Spec spec = ExpressionFunctionRegistry.get(call.name()); - if (spec.rule() == ExpressionFunctionRegistry.Rule.COALESCE) { - for (int i = 0; i < children.size() - 1; i++) { text("IFNULL("); append(children.get(i)); text(", "); } - append(children.get(children.size() - 1)); - for (int i = 0; i < children.size() - 1; i++) text(")"); - } else if (spec.rule() == ExpressionFunctionRegistry.Rule.NULLIF) { - text("(IF ("); append(children.get(0)); text(" = "); append(children.get(1)); - text(") THEN NULL ELSE "); append(children.get(0)); text(" END)"); - } else { - text((call.distinct() ? "COUNTD" : spec.target()) + "("); - if (call.name().equals("POSITION")) { - append(children.get(1)); text(", "); append(children.get(0)); - } else { - for (int i = 0; i < children.size(); i++) { if (i > 0) text(", "); append(children.get(i)); } - } - text(")"); - } - } - } -} diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaMetricExpressionParser.java similarity index 52% rename from converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java rename to converters/salesforce/src/main/java/org/apache/ossie/converter/TuaMetricExpressionParser.java index 004d08ff..1472c592 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaExpressionParser.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/TuaMetricExpressionParser.java @@ -19,19 +19,19 @@ package org.apache.ossie.converter; -import static org.apache.ossie.converter.ExpressionAst.*; -import static org.apache.ossie.converter.ExpressionTokens.*; +import static org.apache.ossie.converter.MetricExpression.*; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Set; /** Native Tua frontend for the bounded supported grammar; produces the same AST as SQL. */ -final class TuaExpressionParser { +final class TuaMetricExpressionParser { private final List tokens; private int position; private int depth; - TuaExpressionParser(List tokens) { this.tokens = tokens; } + TuaMetricExpressionParser(List tokens) { this.tokens = tokens; } Node parse() { Node node = expression(); @@ -95,7 +95,7 @@ private Node primary() { if (take("NULL")) return new Literal(null); if (at("TRUE") || at("FALSE")) return new Literal(Boolean.valueOf(next().text())); Token token = next(); - if (token.kind() == Kind.NUMBER) return new Literal(ExpressionTokens.number(token.text())); + if (token.kind() == Kind.NUMBER) return new Literal(number(token.text())); if (token.kind() == Kind.STRING) return new Literal(token.text()); if (token.kind() != Kind.WORD && token.kind() != Kind.IDENTIFIER) { throw error("expected a value, found '" + token.text() + "'"); @@ -107,11 +107,8 @@ private Node primary() { List parts = new ArrayList<>(); addIdentifier(parts, token); while (take(".")) addIdentifier(parts, next()); - // One bracketed name is a semantic metric reference; the model resolver - // distinguishes it from an unknown or ambiguous field. Physical fields - // still require dataset qualification in the field resolver. - if (parts.size() > 2) throw error("TABLEAU references must use [metric] or [dataset].[field] notation"); - return new Field(new ExpressionCompiler.Reference(parts, true)); + if (parts.size() != 2) throw error("TABLEAU fields must use [dataset].[field] notation"); + return new Field(parts, true); } private void addIdentifier(List parts, Token token) { if (token.kind() != Kind.IDENTIFIER || !token.bracket()) { @@ -146,4 +143,88 @@ private boolean at(String text) { private IllegalArgumentException error(String message) { return new IllegalArgumentException("TABLEAU at character " + (peek().offset() + 1) + ": " + message); } + + // Preflight bounds SQL parsing too; native parsing also consumes these tokens. + enum Kind { WORD, IDENTIFIER, STRING, NUMBER, SYMBOL, END } + record Token(Kind kind, String text, int offset, boolean bracket) {} + static List tokenize(String text, String dialect) { + if (text.length() > 32768) throw new IllegalArgumentException("expression exceeds 32768 characters"); + List tokens = new ArrayList<>(); + int nesting = 0; + for (int i = 0; i < text.length();) { + char c = text.charAt(i); + if (Character.isWhitespace(c)) { i++; continue; } + int start = i; + if (c == '\'' || c == '"' || c == '[') { + if (c == '[' && dialect.equals("SNOWFLAKE")) throw lexical(dialect, i, "use double-quoted SQL identifiers"); + boolean string = c == '\'' || (c == '"' && dialect.equals("TABLEAU")); + char end = c == '[' ? ']' : c; + StringBuilder value = new StringBuilder(); + boolean closed = false; + i++; + while (i < text.length()) { + char part = text.charAt(i++); + if (part == end) { + if (i < text.length() && text.charAt(i) == end) { value.append(end); i++; } + else { closed = true; break; } + } else { + if (Character.isISOControl(part) || (string && part == '\\')) { + throw lexical(dialect, i - 1, "control characters and backslash string escapes are unsupported"); + } + value.append(part); + } + } + if (!closed) throw lexical(dialect, start, "unterminated quoted value"); + tokens.add(new Token(string ? Kind.STRING : Kind.IDENTIFIER, value.toString(), start, c == '[')); + } else if (Character.isDigit(c) || (c == '.' && i + 1 < text.length() && Character.isDigit(text.charAt(i + 1)))) { + i++; + while (i < text.length() && (Character.isDigit(text.charAt(i)) || text.charAt(i) == '.')) i++; + if (i < text.length() && (text.charAt(i) == 'e' || text.charAt(i) == 'E')) { + i++; + if (i < text.length() && (text.charAt(i) == '+' || text.charAt(i) == '-')) i++; + while (i < text.length() && Character.isDigit(text.charAt(i))) i++; + } + tokens.add(new Token(Kind.NUMBER, text.substring(start, i), start, false)); + } else if (Character.isLetter(c) || c == '_') { + i++; + while (i < text.length() && (Character.isLetterOrDigit(text.charAt(i)) || text.charAt(i) == '_' || text.charAt(i) == '$')) i++; + tokens.add(new Token(Kind.WORD, text.substring(start, i), start, false)); + } else { + if (i + 1 < text.length() && (text.startsWith("--", i) || text.startsWith("/*", i))) { + throw lexical(dialect, i, "comments are unsupported in metric expressions"); + } + String symbol = String.valueOf(c); + if (i + 1 < text.length() && Set.of("<=", ">=", "<>", "!=").contains(text.substring(i, i + 2))) { + symbol = text.substring(i, i + 2); + i++; + } + if (!"()+-*/.,=<>!".contains(String.valueOf(c))) throw lexical(dialect, start, "unsupported character '" + c + "'"); + tokens.add(new Token(Kind.SYMBOL, symbol, start, false)); + i++; + } + Token added = tokens.get(tokens.size() - 1); + if (added.kind() == Kind.NUMBER) number(added.text()); + if (added.kind() == Kind.SYMBOL && added.text().equals("(") && ++nesting > 128) { + throw lexical(dialect, start, "expression nesting exceeds 128 levels"); + } + if (added.kind() == Kind.SYMBOL && added.text().equals(")")) nesting--; + if (tokens.size() > 8192) throw lexical(dialect, start, "too many expression tokens"); + } + if (nesting > 0) throw lexical(dialect, text.length(), "expected )"); + tokens.add(new Token(Kind.END, "end of expression", text.length(), false)); + return tokens; + } + + private static IllegalArgumentException lexical(String dialect, int offset, String message) { + return new IllegalArgumentException(dialect + " at character " + (offset + 1) + ": " + message); + } + static BigDecimal number(String text) { + BigDecimal value; + try { value = new BigDecimal(text); } + catch (NumberFormatException e) { throw new IllegalArgumentException("invalid numeric literal '" + text + "'"); } + if (Math.abs((long) value.scale()) > 1000 || value.precision() > 1000) { + throw new IllegalArgumentException("numeric literal is too large"); + } + return value; + } } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java index abac7ddf..95fef55c 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/pipeline/PipelineStep.java @@ -20,7 +20,6 @@ package org.apache.ossie.converter.pipeline; import java.util.Map; -import org.apache.ossie.converter.ConversionContext; /** * Base interface for pipeline steps. @@ -36,9 +35,4 @@ public interface PipelineStep { * @param mappings Property mappings */ void execute(Map sourceData, Map outputData, Map mappings); - - /** Execute with the field catalog and other state belonging to this model only. */ - default void execute(ConversionContext context, Map mappings) { - execute(context.sourceData(), context.outputData(), mappings); - } } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java index 9464df4d..93528956 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricCliTest.java @@ -21,21 +21,12 @@ import static org.junit.jupiter.api.Assertions.*; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.ossie.app.OssieSalesforceConverter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; class MetricCliTest { @TempDir @@ -66,150 +57,4 @@ void reportsMetricFailureToStderrWithoutWritingAModel() throws Exception { process.destroyForcibly(); } } - - @Test - void bindingsFlagWritesCompleteModelWithPhysicalOverrides() throws Exception { - Path input = inputFixture(); - String before = Files.readString(input); - Path bindings = write("bindings.yaml", """ - models: - Customer_Orders_Model: - dataspace: production - datasets: - Orders: - dataObjectName: OrdersProduction__dll - dataObjectType: Dlo - fields: - amount: NetRevenue__c - """); - CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); - assertEquals(0, result.exitCode(), result.stderr()); - JsonNode output = new ObjectMapper().readTree(Files.readString(directory.resolve("Customer_Orders_Model.json"))); - assertEquals("production", output.get("dataspace").asText()); - JsonNode orders = find(output.get("semanticDataObjects"), "Orders"); - assertEquals("OrdersProduction__dll", orders.get("dataObjectName").asText()); - assertEquals("NetRevenue__c", find(orders.get("semanticMeasurements"), "amount").get("dataObjectFieldName").asText()); - assertEquals("SUM([Orders].[amount])", find(output.get("semanticCalculatedMeasurements"), "total_revenue").get("expression").asText()); - assertEquals(before, Files.readString(input)); - } - - @ParameterizedTest - @ValueSource(strings = { - "models: {}\nmodels: {}", - "models: {Customer_Orders_Model: {expression: 'SUM(amount)'}}", - "models: {}\n---\nmodels: {Customer_Orders_Model: {dataspace: hidden}}" - }) - void invalidBindingsDocumentReportsInputErrorAndWritesNothing(String content) throws Exception { - Path input = inputFixture(); - Path bindings = write("bindings.yaml", content); - CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); - assertEquals(2, result.exitCode(), result.stderr()); - assertTrue(result.stderr().contains("Invalid Salesforce bindings"), result.stderr()); - assertFalse(result.stderr().contains("Exception in thread"), result.stderr()); - assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); - } - - @Test - void schemaInvalidBindingReportsConversionErrorWithoutStackTrace() throws Exception { - Path input = inputFixture(); - Path bindings = write("bindings.yaml", """ - models: - Customer_Orders_Model: - datasets: - Orders: {dataObjectType: NotANativeObjectType} - """); - CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); - assertEquals(3, result.exitCode(), result.stderr()); - assertTrue(result.stderr().contains("dataObjectType"), result.stderr()); - assertTrue(result.stderr().startsWith("Error:"), result.stderr()); - assertFalse(result.stderr().contains("Exception in thread"), result.stderr()); - assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); - } - - @Test - void rejectsBindingsFlagInReverseDirectionBeforeReadingBindings() throws Exception { - Path input = write("source.json", Files.readString(Path.of("src/test/resources/examples/salesforceToOssie.json"))); - CliResult result = run("toOssie", input.toString(), "--bindings", directory.resolve("nonexistent.yaml").toString()); - assertEquals(2, result.exitCode(), result.stderr()); - assertTrue(result.stderr().contains("Expected toSF"), result.stderr()); - try (var paths = Files.list(directory)) { - assertFalse(paths.anyMatch(path -> path.toString().endsWith(".yaml"))); - } - } - - @Test - void rejectsMissingBindingsPathAndUnknownFlags() throws Exception { - Path input = inputFixture(); - CliResult missingArgument = run("toSF", input.toString(), "--bindings"); - assertEquals(2, missingArgument.exitCode(), missingArgument.stderr()); - assertTrue(missingArgument.stderr().contains("Expected toSF"), missingArgument.stderr()); - CliResult missingFile = run("toSF", input.toString(), "--bindings", directory.resolve("missing.yaml").toString()); - assertEquals(2, missingFile.exitCode(), missingFile.stderr()); - assertTrue(missingFile.stderr().contains("Cannot read Salesforce bindings"), missingFile.stderr()); - CliResult unknown = run("toSF", input.toString(), "--mapping", "ignored"); - assertEquals(2, unknown.exitCode(), unknown.stderr()); - assertFalse(Files.exists(directory.resolve("Customer_Orders_Model.json"))); - } - - @Test - void laterModelBindingFailurePreservesExistingOutputsAndWritesNoPartialModels() throws Exception { - ObjectMapper yaml = new ObjectMapper(new YAMLFactory()); - Map document = yaml.readValue(Files.readString( - Path.of("src/test/resources/examples/ossieToSalesforce.yaml")), new TypeReference<>() {}); - @SuppressWarnings("unchecked") Map first = (Map) ((List) document.get("semantic_model")).get(0); - Map second = new ObjectMapper().convertValue(first, new TypeReference<>() {}); - second.put("name", "Other_Model"); - document.put("semantic_model", List.of(first, second)); - Path input = write("input.yaml", yaml.writeValueAsString(document)); - String before = Files.readString(input); - Path existing = write("Customer_Orders_Model.json", "preserve existing model"); - Path bindings = write("bindings.yaml", """ - models: - Other_Model: - datasets: - Orders: - fields: {missing: InvalidColumn__c} - """); - CliResult result = run("toSF", input.toString(), "--bindings", bindings.toString()); - assertEquals(3, result.exitCode(), result.stderr()); - assertTrue(result.stderr().contains("Other_Model"), result.stderr()); - assertTrue(result.stderr().contains("missing"), result.stderr()); - assertEquals("preserve existing model", Files.readString(existing)); - assertFalse(Files.exists(directory.resolve("Other_Model.json"))); - assertEquals(before, Files.readString(input)); - } - - private Path inputFixture() throws Exception { - return write("input.yaml", Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml"))); - } - - private Path write(String name, String content) throws Exception { - Path path = directory.resolve(name); - Files.writeString(path, content); - return path; - } - - private record CliResult(int exitCode, String stderr) {} - - private CliResult run(String... arguments) throws Exception { - List command = new ArrayList<>(List.of( - Path.of(System.getProperty("java.home"), "bin", "java").toString(), - "-cp", System.getProperty("java.class.path"), OssieSalesforceConverter.class.getName())); - command.addAll(List.of(arguments)); - Path stderr = directory.resolve("stderr.txt"); - Process process = new ProcessBuilder(command).redirectError(stderr.toFile()) - .redirectOutput(directory.resolve("stdout.txt").toFile()).start(); - try { - assertTrue(process.waitFor(30, TimeUnit.SECONDS), "CLI did not terminate"); - return new CliResult(process.exitValue(), Files.readString(stderr)); - } finally { - process.destroyForcibly(); - } - } - - private static JsonNode find(JsonNode items, String name) { - for (JsonNode item : items) if (name.equals(item.path("apiName").asText())) return item; - throw new AssertionError("Missing exported item " + name); - } - } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java index c16bd761..b382efc3 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/MetricExportIntegrationTest.java @@ -25,7 +25,6 @@ import org.apache.ossie.converter.ConversionDirection; import org.apache.ossie.converter.Converter; import org.apache.ossie.converter.ConverterFactory; -import org.apache.ossie.converter.SalesforceBindings; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.exception.ValidationException; import org.apache.ossie.validator.SchemaValidator; @@ -72,7 +71,6 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() { - assumeTrue(salesforceSchemaExists, "Salesforce schema is required; see README setup instructions"); assumeTrue(ossieSchemaExists, "Ossie schema is required; see README setup instructions"); converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); } @@ -235,18 +233,23 @@ void fieldsFromAnotherSemanticModelCannotSatisfyAMetricReference() throws Except assertTrue(error.getMessage().contains("orders.profit"), error.getMessage()); } - @ParameterizedTest - @ValueSource(strings = {"profit__c + 1", "profit__c+1"}) - void derivedSqlFieldIsExportedAndAvailableToMetricsRegardlessOfWhitespace(String expression) throws Exception { - Map source = model("sales", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); + @Test + void declaredButOmittedCalculatedSqlFieldCannotSatisfyAMetricReference() throws Exception { + Map source = model("sales", List.of()); Map calculated = field("adjusted", "Decimal"); - calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", expression)))); + calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", "profit__c + 1")))); items(source, "datasets").get(0).put("fields", List.of(field("profit", "Decimal"), calculated)); Map output = convertOne(source); assertEquals(List.of("profit"), items(items(output, "semanticDataObjects").get(0), "semanticMeasurements").stream().map(item -> item.get("apiName")).toList()); - assertEquals(1, items(output, "semanticCalculatedDimensions").size()); - assertEquals("SUM(([orders].[profit] + 1))", measurements(output).get(0).get("expression")); + + source.put("metrics", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); + String input = document(List.of(source)); + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(input)); + + assertTrue(error.getMessage().contains("adjusted_total"), error.getMessage()); + assertTrue(error.getMessage().contains("orders.adjusted"), error.getMessage()); + assertTrue(error.getMessage().contains("not exported"), error.getMessage()); } @Test @@ -311,107 +314,37 @@ void translatedComposedMetricsValidateAgainstTheSalesforceSchema() throws Except assertThrows(ValidationException.class, () -> validator.validate(output)); } - @Test - void metricDependenciesResolveForwardReferencesAndRetainDeclarationOrder() throws Exception { - Map output = convertOne(model("sales", List.of( - metric("margin", "SNOWFLAKE", "total_profit / NULLIF(total_sales, 0)"), - metric("total_profit", "ANSI_SQL", "SUM(orders.profit)"), - metric("total_sales", "ANSI_SQL", "SUM(orders.revenue)")))); - assertEquals(List.of("margin", "total_profit", "total_sales"), measurements(output).stream().map(m -> m.get("apiName")).toList()); - String formula = measurements(output).get(0).get("expression").toString(); - assertTrue(formula.contains("SUM([orders].[profit])"), formula); - assertTrue(formula.contains("SUM([orders].[revenue])"), formula); - assertFalse(formula.contains("total_sales")); - } - - @Test - void nativeMetricMetadataSurvivesWhileFormulaMetadataIsCompiled() throws Exception { - Map metric = metric("money", "ANSI_SQL", "SUM(orders.profit)"); - metric.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - JSON.writeValueAsString(Map.of("label", "Profit in USD", "decimalPlace", 2, "dataType", "Currency", - "expression", "stale", "syntax", "Salesforce", "aggregationType", "Avg", "level", "Row"))))); - Map value = measurements(convertOne(model("sales", List.of(metric)))).get(0); - assertEquals("Profit in USD", value.get("label")); - assertEquals(2, value.get("decimalPlace")); - assertEquals("Currency", value.get("dataType")); - assertEquals("SUM([orders].[profit])", value.get("expression")); - assertEquals("Tua", value.get("syntax")); - assertEquals("UserAgg", value.get("aggregationType")); - assertEquals("AggregateFunction", value.get("level")); - } - - @Test - void incompatibleNativeMetricTypeFails() throws Exception { - Map metric = metric("money", "ANSI_SQL", "SUM(orders.profit)"); - metric.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataType\":\"Text\"}"))); - var error = assertThrows(ConversionException.class, () -> convertOne(model("sales", List.of(metric)))); - assertTrue(error.getMessage().contains("incompatible Salesforce dataType Text"), error.getMessage()); - } - - @Test - void outputSchemaIsEnforcedByPublicConversionAndWritesNothingOnFailure() throws Exception { - Map source = model("sales", List.of()); - source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataspace\":42}"))); - Path input = temporaryDirectory.resolve("bad-output.yaml"); - Files.writeString(input, document(List.of(source))); - Path output = Files.createDirectory(temporaryDirectory.resolve("output")); - assertThrows(ValidationException.class, () -> converter.convert(input, output)); - try (var files = Files.list(output)) { assertEquals(0, files.count()); } - } - - @Test - void sameConverterCanRecoverAfterFailureWithoutLeakingDependencyState() throws Exception { - Map bad = model("sales", List.of(metric("a", "ANSI_SQL", "b + 1"), metric("b", "ANSI_SQL", "a + 1"))); - assertThrows(ConversionException.class, () -> convertOne(bad)); - Map good = model("sales", List.of(metric("a", "ANSI_SQL", "b + 1"), metric("b", "ANSI_SQL", "SUM(orders.profit)"))); - assertEquals(convertOne(good), convertOne(good)); - } - - @Test - void bindingsRetargetPhysicalObjectsWithoutChangingTheOsiDocumentOrDerivedReferences() throws Exception { - Map source = model("sales", List.of(metric("total", "ANSI_SQL", "SUM(orders.adjusted)"))); + @ParameterizedTest + @ValueSource(strings = {"profit__c+1", "profit__c-1", "profit__c=1", "1"}) + void rejectsDerivedExpressionsMisclassifiedAsPhysicalColumns(String expression) throws Exception { Map calculated = field("adjusted", "Decimal"); - calculated.put("expression", Map.of("dialects", List.of(dialect("SNOWFLAKE", "profit__c + 1")))); + calculated.put("expression", Map.of("dialects", List.of(dialect("ANSI_SQL", expression)))); + Map source = model("sales", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(orders.adjusted)"))); items(source, "datasets").get(0).put("fields", List.of(field("profit", "Decimal"), calculated)); - String document = document(List.of(source)); - SalesforceBindings bindings = SalesforceBindings.fromString(""" - models: - sales: - dataspace: analytics - datasets: - orders: - dataObjectName: OrdersProduction__dll - dataObjectType: Dlo - fields: - profit: NetProfit__c - """); - Converter bound = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings); - Map output = parse(bound.convert(document).get(0)); - assertEquals("analytics", output.get("dataspace")); - Map dataset = items(output, "semanticDataObjects").get(0); - assertEquals("OrdersProduction__dll", dataset.get("dataObjectName")); - assertEquals("NetProfit__c", items(dataset, "semanticMeasurements").get(0).get("dataObjectFieldName")); - assertEquals("SUM(([orders].[profit] + 1))", measurements(output).get(0).get("expression")); - assertEquals(document, document(List.of(source))); - assertEquals(output, parse(bound.convert(document).get(0))); - } - - @Test - void unusedDirectFieldCannotSilentlyChangeDatatype() throws Exception { - Map source = model("sales", List.of()); - items(items(source, "datasets").get(0), "fields").get(0).put("custom_extensions", - List.of(Map.of("vendor_name", "SALESFORCE", "data", "{\"dataType\":\"Text\"}"))); var error = assertThrows(ConversionException.class, () -> convertOne(source)); - assertTrue(error.getMessage().contains("conflicts"), error.getMessage()); + assertTrue(error.getMessage().contains("adjusted_total"), error.getMessage()); + assertTrue(error.getMessage().contains("direct physical binding"), error.getMessage()); } - @ParameterizedTest - @ValueSource(strings = {"Time", "Opaque"}) - void unsupportedDirectDatatypeNeedsExplicitNativeMapping(String datatype) throws Exception { - Map source = model("sales", List.of()); - items(items(source, "datasets").get(0), "fields").get(0).put("datatype", datatype); + @Test + void metricCannotUseJoinCriteriaCorruptedByInheritedRelationshipFiltering() throws Exception { + Map source = model("sales", List.of(metric("combined", "ANSI_SQL", + "SUM(orders.profit) + SUM(returns.profit)"))); + Map returns = new LinkedHashMap<>(items(source, "datasets").get(0)); + returns.put("name", "returns"); + returns.put("source", "returns__dll"); + source.put("datasets", List.of(items(source, "datasets").get(0), returns)); + Map valid = Map.of("name", "orders_returns", "from", "orders", "to", "returns", + "from_columns", List.of("customer_id"), "to_columns", List.of("customer_id")); + source.put("relationships", List.of(valid)); + assertTrue(measurements(convertOne(source)).get(0).get("expression").toString().contains("[returns].[profit]")); + + source.put("relationships", List.of(Map.of("name", "removed_first", "from", "orders", "to", "returns", + "from_columns", List.of("missing"), "to_columns", List.of("customer_id")), valid)); var error = assertThrows(ConversionException.class, () -> convertOne(source)); - assertTrue(error.getMessage().contains("no safe Salesforce mapping"), error.getMessage()); + assertTrue(error.getMessage().contains("combined"), error.getMessage()); + assertTrue(error.getMessage().contains("orders_returns"), error.getMessage()); + assertTrue(error.getMessage().contains("join key correspondence"), error.getMessage()); } private Map convertOne(Map model) throws IOException { diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index f11622af..99cd754f 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -86,7 +86,6 @@ static void checkSchemaAvailability() { @BeforeEach void setUp() throws IOException { - assumeTrue(salesforceSchemaExists, "Salesforce schema is required; see README setup instructions."); assumeTrue(ossieSchemaExists, "Ossie schema file is required but not found. See README for setup instructions."); converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); @@ -225,15 +224,11 @@ void testCalculatedFieldDetection() throws Exception { Map ansiModel = jsonMapper.readValue(ansiResults.get(0), new TypeReference>() {}); List> ansiCalcDimensions = (List>) ansiModel.get("semanticCalculatedDimensions"); - assertNotNull(ansiCalcDimensions); - assertEquals(2, ansiCalcDimensions.size()); - assertTrue(ansiCalcDimensions.stream().allMatch(field -> "Tua".equals(field.get("syntax")))); - assertTrue(ansiCalcDimensions.stream().anyMatch(field -> field.get("expression").toString().contains("MID("))); - assertTrue(ansiCalcDimensions.stream().anyMatch(field -> field.get("expression").toString().contains("YEAR("))); + assertNull(ansiCalcDimensions, "ANSI_SQL dialect: no semanticCalculatedDimensions"); } @Test - void testAllDeclaredRelationshipsAreExported() throws Exception { + void testInvalidRelationshipsFiltered() throws Exception { List results = converter.convert(ossieYaml); Map sfModel = jsonMapper.readValue(results.get(0), new TypeReference>() {}); @@ -250,21 +245,6 @@ void testAllDeclaredRelationshipsAreExported() throws Exception { assertTrue(hasValidOrdersProducts, "Orders_Products should be included"); } - @Test - void testCalculatedRelationshipKeyFailsInsteadOfDroppingTheRelationship() { - String invalid = ossieYaml.replace(" metrics:", """ - - name: Orders_ByYear - from: Orders - to: Products - from_columns: [order_year] - to_columns: [product_id] - metrics:""".indent(4).stripTrailing()); - Exception error = assertThrows(org.apache.ossie.exception.ConversionException.class, - () -> converter.convert(invalid)); - assertTrue(error.getMessage().contains("Orders_ByYear"), error.getMessage()); - assertTrue(error.getMessage().contains("order_year"), error.getMessage()); - } - @Test void testCustomExtensionsRestoration() throws Exception { List results = converter.convert(ossieYaml); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java index 665944fc..ffc77647 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java @@ -178,8 +178,8 @@ void testFieldMapping() throws Exception { List> dialects = (List>) expression.get("dialects"); assertNotNull(dialects); assertEquals(1, dialects.size()); - assertEquals("ANSI_SQL", dialects.get(0).get("dialect")); - assertEquals("\"customer_id__c\"", dialects.get(0).get("expression")); + assertEquals("TABLEAU", dialects.get(0).get("dialect")); + assertEquals("customer_id__c", dialects.get(0).get("expression")); } @Test diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java deleted file mode 100644 index ee817f47..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/ConstantFieldMetricTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class ConstantFieldMetricTest { - private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); - private static final ObjectMapper JSON = new ObjectMapper(); - - @ParameterizedTest - @ValueSource(strings = {"ANSI_SQL", "SNOWFLAKE", "TABLEAU"}) - void rejectsSummingConstantRowFieldsWhoseDatasetWouldDisappear(String dialect) throws Exception { - Map document = fixture(); - Map model = model(document); - for (String dataset : List.of("Orders", "Products")) { - addField(model, dataset, field("one_per_row", dialect, "1")); - } - // Each individual metric needs a different row population; SUM(1) cannot express this. - for (String dataset : List.of("Orders", "Products")) { - String reference = dialect.equals("TABLEAU") ? "[" + dataset + "].[one_per_row]" : dataset + ".one_per_row"; - model.put("metrics", List.of(metric("row_population", dialect, "SUM(" + reference + ")"))); - ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); - assertTrue(error.getMessage().contains("Metric 'row_population'"), error.getMessage()); - assertTrue(error.getMessage().contains(dataset + ".one_per_row"), error.getMessage()); - assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); - } - } - - @ParameterizedTest - @ValueSource(strings = {"COUNT", "COUNTD", "MIN", "MAX", "AVG"}) - void doesNotAllowOtherAggregatesToBypassConstantFieldScopeChecks(String function) throws Exception { - Map document = fixture(); - Map model = model(document); - addField(model, "Orders", field("one_per_row", "TABLEAU", "1 + 0")); - model.put("metrics", List.of(metric("count_rows", "TABLEAU", function + "([Orders].[one_per_row])"))); - ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); - assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); - } - - @ParameterizedTest - @ValueSource(strings = {"ANSI_SQL", "SNOWFLAKE", "TABLEAU"}) - void stillExportsStandaloneConstantFieldsAndConstantMetrics(String dialect) throws Exception { - Map document = fixture(); - Map model = model(document); - addField(model, "Orders", field("constant", dialect, "1 + 1")); - addField(model, "Products", field("constant", dialect, "2")); - model.put("metrics", List.of(metric("fixed_value", dialect, "42"))); - Map output = convert(document); - assertEquals("42", items(output, "semanticCalculatedMeasurements").get(0).get("expression")); - assertEquals(2, items(output, "semanticCalculatedDimensions").stream() - .filter(item -> List.of("Orders__constant", "Products__constant").contains(item.get("apiName"))).count()); - } - - @Test - void derivedRowExpressionCanCombineAConstantFieldWithAPhysicalField() throws Exception { - Map document = fixture(); - Map model = model(document); - addField(model, "Orders", field("constant_one", "ANSI_SQL", "1")); - Map amountPlusOne = field("amount_plus_one", "ANSI_SQL", "amount + constant_one"); - amountPlusOne.put("datatype", "Decimal"); - addField(model, "Orders", amountPlusOne); - model.put("metrics", List.of(metric("adjusted_total", "ANSI_SQL", "SUM(Orders.amount_plus_one)"))); - String expression = items(convert(document), "semanticCalculatedMeasurements").get(0).get("expression").toString(); - assertTrue(expression.contains("[Orders].[amount]"), expression); - assertTrue(expression.contains("+ 1"), expression); - } - - @Test - void constantDependenciesCannotSmuggleDatasetScopeThroughAnAlias() throws Exception { - Map document = fixture(); - Map model = model(document); - addField(model, "Orders", field("constant_one", "ANSI_SQL", "1")); - addField(model, "Orders", field("constant_alias", "TABLEAU", "[Orders].[constant_one] + 0")); - model.put("metrics", List.of(metric("total", "ANSI_SQL", "SUM(Orders.constant_alias)"))); - ConversionException error = assertThrows(ConversionException.class, () -> convert(document)); - assertTrue(error.getMessage().contains("without a physical dataset anchor"), error.getMessage()); - } - - private static Map field(String name, String dialect, String expression) { - return new LinkedHashMap<>(Map.of("name", name, "datatype", "Integer", "expression", - Map.of("dialects", List.of(Map.of("dialect", dialect, "expression", expression))))); - } - - private static Map metric(String name, String dialect, String expression) { - Map metric = field(name, dialect, expression); - metric.put("datatype", "Decimal"); - return metric; - } - - private static void addField(Map model, String datasetName, Map field) { - Map dataset = items(model, "datasets").stream() - .filter(item -> datasetName.equals(item.get("name"))).findFirst().orElseThrow(); - List> fields = new ArrayList<>(items(dataset, "fields")); - fields.add(field); - dataset.put("fields", fields); - } - - private static Map fixture() throws Exception { - return YAML.readValue(Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")), new TypeReference<>() {}); - } - - private static Map model(Map document) { - return items(document, "semantic_model").get(0); - } - - private static Map convert(Map document) throws Exception { - String output = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE) - .convert(YAML.writeValueAsString(document)).get(0); - return JSON.readValue(output, new TypeReference<>() {}); - } - - @SuppressWarnings("unchecked") - private static List> items(Map map, String key) { - return (List>) map.getOrDefault(key, List.of()); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java deleted file mode 100644 index 1145ce57..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/CustomExtensionHandlerTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.NullSource; -import org.junit.jupiter.params.provider.ValueSource; - -class CustomExtensionHandlerTest { - private final ObjectMapper json = new ObjectMapper(); - private final CustomExtensionHandler handler = new CustomExtensionHandler(json); - - @ParameterizedTest - @ValueSource(strings = {"{", "{} {}", "{} null", "{\"label\":\"a\",\"label\":\"b\"}", - "{\"nested\":{\"x\":1,\"x\":2}}", "null", "[]", "42", "\"text\""}) - void rejectsMalformedAmbiguousOrNonObjectJsonWithoutWritingPartialProperties(String data) { - Map target = new LinkedHashMap<>(Map.of("apiName", "owner")); - ConversionException error = assertThrows(ConversionException.class, - () -> handler.restoreSalesforceCustomExtension(target, item("owner", data))); - assertTrue(error.getMessage().contains("owner"), error.getMessage()); - assertTrue(error.getMessage().contains("JSON object"), error.getMessage()); - assertEquals(Map.of("apiName", "owner"), target); - } - - @ParameterizedTest - @NullSource - @ValueSource(ints = {1}) - void rejectsMissingOrNonStringData(Object data) { - ConversionException error = assertThrows(ConversionException.class, - () -> handler.restoreSalesforceCustomExtension(new LinkedHashMap<>(), item("owner", data))); - assertTrue(error.getMessage().contains("owner")); - assertTrue(error.getMessage().contains("encoded as a string")); - } - - @Test - void preservesNativePropertiesAndCorePrecedenceWithoutChangingSharedMapper() { - Map target = new LinkedHashMap<>(Map.of("label", "core")); - handler.restoreSalesforceCustomExtension(target, - item("owner", "{\"label\":\"native\",\"description\":\"kept\",\"nested\":{\"x\":1}}")); - assertEquals("core", target.get("label")); - assertEquals("kept", target.get("description")); - assertEquals(Map.of("x", 1), target.get("nested")); - assertFalse(json.isEnabled(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)); - assertFalse(json.isEnabled(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)); - } - - @Test - void ignoresOtherVendorData() { - Map source = new LinkedHashMap<>(Map.of("name", "owner", "custom_extensions", - List.of(Map.of("vendor_name", "SNOWFLAKE", "data", "not Salesforce JSON")))); - Map target = new LinkedHashMap<>(); - assertDoesNotThrow(() -> handler.restoreSalesforceCustomExtension(target, source)); - assertTrue(target.isEmpty()); - } - - @ParameterizedTest - @ValueSource(strings = {"model", "dataset", "field", "metric"}) - void publicConversionRejectsMalformedMetadataAtEveryOwnerLevel(String ownerKind) throws Exception { - Map field = new LinkedHashMap<>(Map.of("name", "amount", "datatype", "Decimal", - "expression", dialect("amount__c"))); - Map dataset = item("orders", "{\"dataObjectType\":\"Dlo\"}"); - dataset.put("source", "orders__dll"); - dataset.put("fields", List.of(field)); - Map metric = new LinkedHashMap<>(Map.of("name", "total", "datatype", "Decimal", - "expression", dialect("SUM(orders.amount)"))); - Map model = item("sales", "{\"dataspace\":\"default\"}"); - model.put("datasets", List.of(dataset)); - model.put("metrics", List.of(metric)); - Map owner = switch (ownerKind) { - case "model" -> model; - case "dataset" -> dataset; - case "field" -> field; - default -> metric; - }; - owner.put("custom_extensions", item("ignored", "{").get("custom_extensions")); - String input = json.writeValueAsString(Map.of("version", "0.2.0.dev0", "semantic_model", List.of(model))); - ConversionException error = assertThrows(ConversionException.class, - () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); - assertTrue(error.getMessage().contains((String) owner.get("name")), error.getMessage()); - assertTrue(error.getMessage().contains("JSON"), error.getMessage()); - } - - private static Map dialect(String expression) { - return Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", expression))); - } - - private static Map item(String name, Object data) { - Map extension = new LinkedHashMap<>(); - extension.put("vendor_name", "SALESFORCE"); - extension.put("data", data); - return new LinkedHashMap<>(Map.of("name", name, "custom_extensions", List.of(extension))); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java deleted file mode 100644 index ab4ebacb..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/ExpressionCompilerTest.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class ExpressionCompilerTest { - private static final Map TYPES = Map.of( - "amount", "Decimal", "quantity", "Integer", "email", "String", "active", "Boolean", - "ordered", "Date", "timestamp", "DateTime", "zoned", "DateTimeTz"); - private static ExpressionCompiler.Binding resolve(ExpressionCompiler.Reference reference) { - String field = reference.parts().getLast().text(); - String datatype = TYPES.get(field); - if (datatype == null) throw new IllegalArgumentException("Unknown field " + field); - return new ExpressionCompiler.Binding("[orders].[" + field + "]", datatype, "orders"); - } - private static ExpressionCompiler.Compiled compile(String source, String dialect) { - return ExpressionCompiler.compile(ExpressionCompiler.parse(source, dialect), ExpressionCompilerTest::resolve); - } - - @Test void parsesWithoutBindingAndRetainsQuotedIdentifiers() { - var parsed = ExpressionCompiler.parse("\"Order.Items\".\"Net Revenue\"", "SNOWFLAKE"); - var reference = ExpressionCompiler.directReference(parsed).orElseThrow(); - assertEquals(List.of(new MetricFieldResolver.Identifier("Order.Items", true), - new MetricFieldResolver.Identifier("Net Revenue", true)), reference.parts()); - assertFalse(reference.tableau()); - assertTrue(ExpressionCompiler.directReference(ExpressionCompiler.parse("(amount)", "ANSI_SQL")).isPresent()); - assertTrue(ExpressionCompiler.directReference(ExpressionCompiler.parse("amount + 1", "ANSI_SQL")).isEmpty()); - } - - @Test void independentlyBindsAndRendersTheSameImmutableParse() { - var parsed = ExpressionCompiler.parse("SUM(amount)", "SNOWFLAKE"); - AtomicInteger calls = new AtomicInteger(); - var first = ExpressionCompiler.compile(parsed, reference -> { - calls.incrementAndGet(); return new ExpressionCompiler.Binding("[one].[net]", "Decimal", "one"); - }); - var second = ExpressionCompiler.compile(parsed, - reference -> new ExpressionCompiler.Binding("[two].[gross]", "Decimal", "two")); - assertEquals(1, calls.get()); - assertEquals("SUM([one].[net])", first.expression()); - assertEquals(Set.of("two"), second.datasets()); - assertEquals("SUM([two].[gross])", second.expression()); - } - - @ParameterizedTest - @ValueSource(strings = {"CAST(amount AS INT)", "amount::INT", "SUM(amount) OVER ()", - "SUM(amount) FILTER (WHERE active)", "(SELECT amount FROM orders)", - "amount IN (1, 2)", "SUM(amount ORDER BY quantity)", "amount(+) = quantity", - "CASE amount WHEN 1 THEN 2 END", "SUM(amount) AS alias", "SUM(amount); SUM(quantity)", - "SUM(amount) trailing", "SELECT amount", "unknown_function(amount)", "COALESCE()"}) - void rejectsShapesWithoutExplicitCapabilities(String source) { - assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); - } - - @ParameterizedTest - @ValueSource(strings = {"SUM(ALL amount)", "SUM(UNIQUE amount)", "SUM(amount IGNORE NULLS)", - "SUM(amount RESPECT NULLS)", "SUM(amount) IGNORE NULLS", - "SUM(amount) KEEP (DENSE_RANK LAST ORDER BY quantity)", "SUM(amount LIMIT 1)", - "SUM(amount HAVING MAX quantity)", "SUM(amount ORDER BY quantity)", - "SUM(amount).attribute", "private_schema.SUM(amount)", "\"SUM\"(amount)", - "N'prefixed'", "email ISNULL", "email NOTNULL", "PRIOR amount = quantity", "!active"}) - void rejectsDialectModifiersInsteadOfSilentlyDiscardingThem(String source) { - assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); - } - - @Test void nativeMetricReferencesAreBoundByTheModelResolver() { - var result = ExpressionCompiler.compile(ExpressionCompiler.parse("[net] + 1", "TABLEAU"), reference -> { - assertTrue(reference.tableau()); - assertEquals(List.of(new MetricFieldResolver.Identifier("net", true)), reference.parts()); - return new ExpressionCompiler.Binding("SUM([orders].[amount])", "Decimal", "orders", ExpressionCompiler.Level.AGGREGATE); - }); - assertEquals("(SUM([orders].[amount]) + 1)", result.expression()); - assertEquals(ExpressionCompiler.Level.AGGREGATE, result.level()); - } - - @Test void escapedQuotedIdentifiersDoNotChangeReferenceBoundaries() { - for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { - var reference = ExpressionCompiler.directReference(ExpressionCompiler.parse("\"a\"\"b.c\".\"d\"\"e\"", dialect)).orElseThrow(); - assertEquals(List.of(new MetricFieldResolver.Identifier("a\"b.c", true), - new MetricFieldResolver.Identifier("d\"e", true)), reference.parts()); - } - var bracket = ExpressionCompiler.directReference(ExpressionCompiler.parse("[a.b].[c]", "ANSI_SQL")).orElseThrow(); - assertTrue(bracket.tableau()); - assertEquals(List.of(new MetricFieldResolver.Identifier("a.b", true), - new MetricFieldResolver.Identifier("c", true)), bracket.parts()); - // JSqlParser does not consume doubled closing brackets. Fail closed, rather - // than resolving a truncated identifier to another field. - assertThrows(IllegalArgumentException.class, - () -> ExpressionCompiler.parse("[a.b].[c]]d]", "ANSI_SQL")); - } - - @Test void preservesNumberPrecisionAndEscapedStringValues() { - assertEquals("0.12345678901234567890123456789", - compile("0.12345678901234567890123456789", "SNOWFLAKE").expression()); - var result = compile("CASE WHEN email = 'O''Brien -- not a comment' THEN 'a''b' ELSE 'x' END", "SNOWFLAKE"); - assertEquals("String", result.datatype()); - assertEquals(ExpressionCompiler.Level.ROW, result.level()); - assertEquals("(IF ([orders].[email] = 'O''Brien -- not a comment') THEN 'a''b' ELSE 'x' END)", result.expression()); - } - - @ParameterizedTest - @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) - void compilesExistingDerivedDateAndEmailDomainExpressions(String dialect) { - var year = compile("YEAR(ordered)", dialect); - assertEquals("YEAR([orders].[ordered])", year.expression()); - assertEquals("Integer", year.datatype()); - var domain = compile("SUBSTRING(email, POSITION('@' IN email) + 1, LENGTH(email))", dialect); - assertEquals("MID([orders].[email], (FIND([orders].[email], '@') + 1), LEN([orders].[email]))", domain.expression()); - assertEquals("String", domain.datatype()); - assertEquals(ExpressionCompiler.Level.ROW, domain.level()); - assertEquals(domain.expression(), compile(domain.expression(), "TABLEAU").expression()); - assertEquals("SUM(YEAR([orders].[ordered]))", compile("SUM(YEAR(ordered))", dialect).expression()); - assertEquals("SUM(LEN([orders].[email]))", compile("SUM(LENGTH(email))", dialect).expression()); - } - - @ParameterizedTest - @ValueSource(strings = {"SUBSTRING(email, 0)", "SUBSTRING(email, -1)", "SUBSTRING(email, quantity)", - "SUBSTRING(email, 1, -1)", "SUBSTRING(email, 1, quantity)", "SUBSTRING(email, 1.5)", - "YEAR(zoned)", "YEAR(email)", "LENGTH(quantity)", "POSITION(1 IN email)"}) - void rejectsScalarDomainsWithoutEquivalentTargetSemantics(String source) { - assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE"), source); - } - - @ParameterizedTest - @ValueSource(strings = {"CEIL(AVG(amount))", "FLOOR(AVG(amount))", "ROUND(AVG(amount))", - "ROUND(AVG(amount), 0)", "ROUND(AVG(amount), -2)"}) - void integralRoundingCanSatisfyAnIntegerMetricDeclaration(String source) { - var result = compile(source, "SNOWFLAKE"); - assertEquals("Integer", result.datatype()); - var metric = Map.of("name", "rounded", "datatype", "Integer", "expression", - Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", source)))); - assertEquals("Integer", MetricExpressionTranslator.compile(metric, ExpressionCompilerTest::resolve).datatype()); - } - - @Test void positivePrecisionDoesNotClaimAnIntegralResult() { - assertEquals("Decimal", compile("ROUND(AVG(amount), 2)", "SNOWFLAKE").datatype()); - assertEquals("Integer", compile("ROUND(SUM(quantity), 2)", "SNOWFLAKE").datatype()); - } - - @Test void boundDerivedRowsAndMetricsPreserveTheirLevelAndLineage() { - var row = ExpressionCompiler.compile(ExpressionCompiler.parse("SUM(net)", "SNOWFLAKE"), - reference -> new ExpressionCompiler.Binding("([orders].[amount] * [orders].[quantity])", "Decimal", "orders")); - assertEquals("SUM(([orders].[amount] * [orders].[quantity]))", row.expression()); - var aggregate = ExpressionCompiler.compile(ExpressionCompiler.parse("ratio + 1", "SNOWFLAKE"), - reference -> new ExpressionCompiler.Binding("(SUM([orders].[amount]) / SUM([costs].[amount]))", - "Decimal", Set.of("orders", "costs"), ExpressionCompiler.Level.AGGREGATE)); - assertEquals(Set.of("orders", "costs"), aggregate.datasets()); - assertEquals(ExpressionCompiler.Level.AGGREGATE, aggregate.level()); - assertThrows(IllegalArgumentException.class, () -> ExpressionCompiler.compile( - ExpressionCompiler.parse("SUM(ratio)", "SNOWFLAKE"), - reference -> new ExpressionCompiler.Binding("SUM([orders].[amount])", "Decimal", "orders", ExpressionCompiler.Level.AGGREGATE))); - } - - @Test void declaredTypeSurvivesAnAllNullMetricForDependentMetrics() { - var metric = Map.of("name", "nullable", "datatype", "Decimal", "expression", - Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "NULL")))); - assertEquals("Decimal", MetricExpressionTranslator.compile(metric, ExpressionCompilerTest::resolve).datatype()); - } - - @Test void limitsInputDepthTokensAndEmittedExpansion() { - assertThrows(IllegalArgumentException.class, () -> compile("(".repeat(129) + "1" + ")".repeat(129), "SNOWFLAKE")); - assertThrows(IllegalArgumentException.class, () -> compile("1+".repeat(5000) + "1", "SNOWFLAKE")); - assertThrows(IllegalArgumentException.class, () -> compile("1".repeat(32769), "SNOWFLAKE")); - String growing = "SUM(amount)"; - for (int i = 0; i < 18; i++) growing = "NULLIF(" + growing + ", 0)"; - String source = growing; - var exception = assertThrows(IllegalArgumentException.class, () -> compile(source, "SNOWFLAKE")); - assertTrue(exception.getMessage().contains("131072"), exception.getMessage()); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java deleted file mode 100644 index 50006ab3..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/FieldExpressionPlanTest.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class FieldExpressionPlanTest { - @ParameterizedTest - @ValueSource(strings = {"amount__c+profit__c", "amount__c + profit__c"}) - void parsesDerivedFieldsIndependentOfWhitespaceAndBindsPhysicalColumns(String expression) { - ConversionContext context = export(dataset("Orders", "warehouse.sales.orders", - field("amount", "Decimal", "amount__c"), field("profit", "Decimal", "profit__c"), - field("derived", "Decimal", expression))); - assertEquals(2, direct(context, "Orders").size()); - Map calc = calculated(context).get(0); - assertEquals("Orders__derived", calc.get("apiName")); - assertEquals("Tua", calc.get("syntax")); - assertEquals("Number", calc.get("dataType")); - assertTrue(calc.get("expression").toString().contains("[Orders].[amount]")); - assertTrue(calc.get("expression").toString().contains("[Orders].[profit]")); - assertFalse(calc.get("expression").toString().contains("__c")); - assertEquals(2, ((List) calc.get("dependencies")).size()); - assertFalse(context.fieldPlan().isDirect("Orders", "derived")); - } - - @ParameterizedTest - @ValueSource(strings = {"cost+tax", "unit(price)", "gross/net", "Order Date", "has\"quote", "SUM", "schema.column"}) - void preservesQuotedPhysicalNamesInsteadOfClassifyingTheirCharacters(String physical) { - ConversionContext context = export(dataset("Orders", "orders", - field("value", "Decimal", "\"" + physical.replace("\"", "\"\"") + "\""))); - assertEquals(physical, direct(context, "Orders").get(0).get("dataObjectFieldName")); - assertTrue(calculated(context).isEmpty()); - } - - @Test - void extractsOnlyPhysicalColumnFromVerifiedSourceQualifier() { - ConversionContext context = export(dataset("Orders", "warehouse.sales.orders", - field("amount", "Decimal", "sales.orders.amount__c"))); - assertEquals("amount__c", direct(context, "Orders").get(0).get("dataObjectFieldName")); - assertFailure("qualifier", dataset("Orders", "warehouse.sales.orders", - field("amount", "Decimal", "another_table.amount__c"))); - } - - @Test - void expandsForwardReferencesAndCachesResolvedRowBindings() { - ConversionContext context = export(dataset("Orders", "orders", - field("gross", "Decimal", "net + tax"), - field("net", "Decimal", "amount - discount"), - field("amount", "Decimal", "amount__c"), - field("discount", "Decimal", "discount__c"), - field("tax", "Decimal", "tax__c"))); - ExpressionCompiler.Binding binding = context.fieldPlan().resolve("Orders", "gross"); - assertSame(binding, context.fieldPlan().resolve("Orders", "gross")); - assertTrue(binding.expression().contains("[Orders].[amount]")); - assertTrue(binding.expression().contains("[Orders].[discount]")); - assertTrue(binding.expression().contains("[Orders].[tax]")); - assertFalse(binding.expression().contains("[net]")); - assertEquals(ExpressionCompiler.Level.ROW, binding.level()); - assertEquals(3, context.fieldPlan().dependencies("Orders", "gross").size()); - } - - @Test - void rejectsCyclesWithWholeDependencyPath() { - assertFailure("Orders.a -> Orders.b -> Orders.a", dataset("Orders", "orders", - field("a", "Decimal", "b + 1"), field("b", "Decimal", "a + 1"))); - assertFailure("dependency cycle", dataset("Orders", "orders", field("a", "Decimal", "a + 1"))); - } - - @Test - void rejectsUnknownOrAmbiguousPhysicalAndSemanticReferences() { - assertFailure("Unknown row field reference", dataset("Orders", "orders", - field("derived", "Decimal", "undeclared + 1"))); - assertFailure("Ambiguous row field reference", dataset("Orders", "orders", - field("one", "Decimal", "amount__c"), field("two", "Decimal", "amount__c"), - field("derived", "Decimal", "amount__c + 1"))); - assertFailure("Ambiguous row field reference", dataset("Orders", "orders", - field("one", "Decimal", "amount"), field("amount", "Decimal", "second__c"), - field("derived", "Decimal", "amount + 1"))); - } - - @Test - void respectsQuotedPhysicalIdentifierCase() { - ConversionContext context = export(dataset("Orders", "orders", - field("amount", "Decimal", "\"mixedCase\""), - field("derived", "Decimal", "\"mixedCase\" + 1"))); - assertTrue(context.fieldPlan().resolve("Orders", "derived").expression().contains("[Orders].[amount]")); - assertFailure("Unknown row field reference", dataset("Orders", "orders", - field("amount", "Decimal", "\"mixedCase\""), - field("derived", "Decimal", "mixedCase + 1"))); - } - - @Test - void rejectsUnsupportedUnusedExpressionsRatherThanOmittingThem() { - assertFailure("Field 'Orders.derived'", dataset("Orders", "orders", - field("amount", "Decimal", "amount__c"), - field("derived", "Decimal", "UNKNOWN_FUNCTION(amount)"))); - } - - @Test - void rejectsAggregateFieldsAndIncompatibleDeclaredResultTypes() { - assertFailure("row expressions", dataset("Orders", "orders", - field("amount", "Decimal", "amount__c"), field("derived", "Decimal", "SUM(amount)"))); - assertFailure("conflicts", dataset("Orders", "orders", - field("amount", "Decimal", "amount__c"), field("derived", "String", "amount + 1"))); - } - - @Test - void infersStringBooleanAndNumericRowResultTypes() { - ConversionContext context = export(dataset("Orders", "orders", - field("amount", "Decimal", "amount__c"), field("name", "String", "name__c"), - field("positive", null, "amount > 0"), - field("category", null, "CASE WHEN amount > 0 THEN 'positive' ELSE name END"), field("constant", null, "42"))); - assertEquals("Boolean", context.fieldPlan().resolve("Orders", "positive").datatype()); - assertEquals("String", context.fieldPlan().resolve("Orders", "category").datatype()); - assertEquals("Integer", context.fieldPlan().resolve("Orders", "constant").datatype()); - } - - @Test - void treatsTableauDirectReferenceAsSemanticAliasNotPhysicalColumn() { - Map alias = field("alias", "Decimal", "[Orders].[amount]"); - alias.put("expression", expression("TABLEAU", "[Orders].[amount]")); - ConversionContext context = export(dataset("Orders", "orders", - field("amount", "Decimal", "amount__c"), alias)); - assertEquals(1, direct(context, "Orders").size()); - assertEquals("[Orders].[amount]", context.fieldPlan().resolve("Orders", "alias").expression()); - assertEquals("Orders__alias", calculated(context).get(0).get("apiName")); - } - - @Test - void rejectsCrossDatasetRowFieldsEvenWhenTheOtherDatasetIsDeclared() { - Map cross = field("cross", "Decimal", "[Returns].[amount] + 1"); - cross.put("expression", expression("TABLEAU", "[Returns].[amount] + 1")); - assertFailure("cross-dataset", dataset("Orders", "orders", cross), - dataset("Returns", "returns", field("amount", "Decimal", "amount__c"))); - assertFailure("qualifier", dataset("Orders", "orders", field("cross", "Decimal", "Returns.amount + 1")), - dataset("Returns", "returns", field("amount", "Decimal", "amount__c"))); - } - - @Test - void usesStableDistinctGlobalNamesAcrossDatasetsAndSanitizationCollisions() { - Map a = dataset("a-b", "a", field("value", "Integer", "1 + 1")); - Map b = dataset("a_b", "b", field("value", "Integer", "2 + 2")); - ConversionContext first = export(a, b); - ConversionContext reversed = export(b, a); - String firstName = first.fieldPlan().calculatedApiName("a-b", "value"); - String secondName = first.fieldPlan().calculatedApiName("a_b", "value"); - assertNotEquals(firstName, secondName); - assertTrue(firstName.matches("[A-Za-z_][A-Za-z0-9_]*")); - assertEquals(firstName, reversed.fieldPlan().calculatedApiName("a-b", "value")); - assertEquals(secondName, reversed.fieldPlan().calculatedApiName("a_b", "value")); - } - - @Test - void reservesMetricNamesBeforeAllocatingCalculatedDimensionNames() { - Map dataset = dataset("Orders", "orders", field("constant", "Integer", "1 + 1")); - Map source = new LinkedHashMap<>(Map.of("datasets", List.of(dataset), - "metrics", List.of(Map.of("name", "Orders__constant")))); - ConversionContext context = exportModel(source); - assertNotEquals("Orders__constant", context.fieldPlan().calculatedApiName("Orders", "constant")); - } - - @Test - void rejectsDuplicateDeclarationsAndMissingTargetDataset() { - assertFailure("Duplicate field", dataset("Orders", "orders", - field("amount", "Decimal", "one__c"), field("amount", "Decimal", "two__c"))); - Map source = Map.of("datasets", List.of(dataset("Orders", "orders", field("a", "Integer", "a")))); - IllegalArgumentException error = assertThrows(IllegalArgumentException.class, - () -> handler().execute(source, new LinkedHashMap<>(), Map.of())); - assertTrue(error.getMessage().contains("was not exported"), error.getMessage()); - } - - @Test - void reverseConversionQuotesPhysicalColumnsAndRoundTripsTheirExactName() { - String physical = "gross/+ \"net\""; - Map sfDataset = new LinkedHashMap<>(Map.of("apiName", "Orders", - "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number", "dataObjectFieldName", physical)))); - Map source = new LinkedHashMap<>(Map.of("semanticDataObjects", List.of(sfDataset))); - Map dataset = new LinkedHashMap<>(Map.of("name", "Orders", "source", "orders")); - Map output = new LinkedHashMap<>(Map.of("datasets", List.of(dataset))); - new FieldMappingHandler(ConversionDirection.SALESFORCE_TO_OSSIE, - new CustomExtensionHandler(new ObjectMapper())).execute(source, output, Map.of()); - ConversionContext context = export(dataset); - assertEquals(physical, direct(context, "Orders").get(0).get("dataObjectFieldName")); - } - - @Test - void preservesQuotedQualifiedPhysicalColumnsAndRejectsWrongQualifierCase() { - String source = "\"Warehouse\".\"SaLes\".\"Order Items\""; - ConversionContext context = export(dataset("Orders", source, - field("amount", "Decimal", source + ".\"Net Value\""), - field("double_amount", "Decimal", "\"SaLes\".\"Order Items\".\"Net Value\" * 2"))); - assertEquals("Net Value", direct(context, "Orders").get(0).get("dataObjectFieldName")); - assertTrue(context.fieldPlan().resolve("Orders", "double_amount").expression().contains("[Orders].[amount]")); - assertFailure("qualifier", dataset("Orders", source, - field("amount", "Decimal", "\"WAREHOUSE\".\"SaLes\".\"Order Items\".\"Net Value\""))); - } - - @Test - void rejectsExcessiveExpansionAndDependencyDepthWithUsefulErrors() { - List> exponential = new ArrayList<>(); - exponential.add(field("f0", "Integer", "1 + 1")); - for (int i = 1; i < 20; i++) { - exponential.add(field("f" + i, "Integer", "f" + (i - 1) + " + f" + (i - 1))); - } - assertFailure("translated expression exceeds 131072 characters", new LinkedHashMap<>(Map.of( - "name", "Orders", "source", "orders", "fields", exponential))); - - List> deep = new ArrayList<>(); - for (int i = 0; i < 130; i++) deep.add(field("f" + i, "Integer", "f" + (i + 1) + " + 1")); - deep.add(field("f130", "Integer", "1 + 1")); - assertFailure("dependency depth exceeds 128", new LinkedHashMap<>(Map.of( - "name", "Orders", "source", "orders", "fields", deep))); - } - - @Test - void physicalBindingOverlayLeavesSourceAndCompiledDependenciesUnchanged() throws Exception { - Map source = new LinkedHashMap<>(Map.of("name", "Retail", "datasets", List.of( - dataset("Orders", "warehouse.orders", field("amount", "Decimal", "amount__c"), - field("derived", "Decimal", "amount__c + 1"))))); - ObjectMapper mapper = new ObjectMapper(); - String original = mapper.writeValueAsString(source); - ConversionContext context = exportModel(source); - ExpressionCompiler.Binding binding = context.fieldPlan().resolve("Orders", "derived"); - List> dependencies = context.fieldPlan().dependencies("Orders", "derived"); - SalesforceBindings.fromString(""" - models: - Retail: - dataspace: prod - datasets: - Orders: - dataObjectName: Orders__dlm - dataObjectType: DataModelObject - fields: - amount: NetRevenue__c - """).apply(source, context.outputData()); - assertEquals(original, mapper.writeValueAsString(source)); - assertEquals("NetRevenue__c", direct(context, "Orders").get(0).get("dataObjectFieldName")); - assertSame(binding, context.fieldPlan().resolve("Orders", "derived")); - assertEquals(dependencies, context.fieldPlan().dependencies("Orders", "derived")); - assertTrue(binding.expression().contains("[Orders].[amount]")); - assertFalse(binding.expression().contains("NetRevenue__c")); - assertEquals(List.of(Map.of("dependentDefinitionApiName", "Orders", "dependentFieldApiName", "amount")), dependencies); - } - - @Test - void calculatedRowMetadataOverridesStaleNativeFormulaProperties() throws Exception { - Map derived = field("derived", "Decimal", "amount + 1"); - derived.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - new ObjectMapper().writeValueAsString(Map.of("level", "AggregateFunction", "syntax", "Salesforce", - "expression", "SUM(stale)", "apiName", "renamed", "label", "Adjusted amount", "dataType", "Currency", - "dependencies", List.of(Map.of("dependentDefinitionApiName", "Wrong", "dependentFieldApiName", "missing"))))))); - ConversionContext context = export(dataset("Orders", "orders", field("amount", "Decimal", "amount__c"), derived)); - Map calc = calculated(context).get(0); - assertEquals("Row", calc.get("level")); - assertEquals("Tua", calc.get("syntax")); - assertEquals("Orders__derived", calc.get("apiName")); - assertEquals("Adjusted amount", calc.get("label")); - assertEquals("Currency", calc.get("dataType")); - assertTrue(calc.get("expression").toString().contains("[Orders].[amount]")); - assertFalse(calc.get("expression").toString().contains("SUM")); - assertEquals(List.of(Map.of("dependentDefinitionApiName", "Orders", "dependentFieldApiName", "amount")), calc.get("dependencies")); - } - - @SafeVarargs - private static void assertFailure(String message, Map... datasets) { - IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () -> export(datasets)); - assertTrue(error.getMessage().contains(message), error.getMessage()); - } - - @SafeVarargs - private static ConversionContext export(Map... datasets) { - return exportModel(new LinkedHashMap<>(Map.of("datasets", List.of(datasets)))); - } - - private static ConversionContext exportModel(Map source) { - List targets = new ArrayList<>(); - for (Object item : (List) source.get("datasets")) { - Map dataset = (Map) item; - targets.add(new LinkedHashMap<>(Map.of("apiName", dataset.get("name")))); - } - Map target = new LinkedHashMap<>(Map.of("semanticDataObjects", targets)); - ConversionContext context = new ConversionContext(source, target); - handler().execute(context, Map.of()); - return context; - } - - private static FieldMappingHandler handler() { - return new FieldMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, - new CustomExtensionHandler(new ObjectMapper())); - } - - @SuppressWarnings("unchecked") - private static List> calculated(ConversionContext context) { - return (List>) context.outputData().getOrDefault("semanticCalculatedDimensions", List.of()); - } - - @SuppressWarnings("unchecked") - private static List> direct(ConversionContext context, String dataset) { - return ((List>) context.outputData().get("semanticDataObjects")).stream() - .filter(item -> dataset.equals(item.get("apiName"))).flatMap(item -> List.of("semanticDimensions", "semanticMeasurements") - .stream().flatMap(key -> ((List>) item.getOrDefault(key, List.of())).stream())).toList(); - } - - @SafeVarargs - private static Map dataset(String name, String source, Map... fields) { - return new LinkedHashMap<>(Map.of("name", name, "source", source, "fields", List.of(fields))); - } - - private static Map field(String name, String datatype, String text) { - Map field = new LinkedHashMap<>(); - field.put("name", name); - if (datatype != null) field.put("datatype", datatype); - field.put("expression", expression("ANSI_SQL", text)); - return field; - } - - private static Map expression(String dialect, String text) { - return Map.of("dialects", List.of(Map.of("dialect", dialect, "expression", text))); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java deleted file mode 100644 index e3e6a965..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricCompilationPlanTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.junit.jupiter.api.Test; - -class MetricCompilationPlanTest { - private static Map metric(String name, String expression) { - return metric(name, expression, "ANSI_SQL"); - } - private static Map metric(String name, String expression, String dialect) { - return new LinkedHashMap<>(Map.of("name", name, "expression", Map.of("dialects", - List.of(Map.of("dialect", dialect, "expression", expression))))); - } - private static MetricCompilationPlan plan(List> metrics) { - Map source = Map.of("metrics", metrics, "datasets", List.of(Map.of("name", "orders", "fields", - List.of(Map.of("name", "profit", "datatype", "Decimal"), Map.of("name", "units", "datatype", "Integer"))))); - Map target = Map.of("semanticDataObjects", List.of(Map.of("apiName", "orders", "semanticMeasurements", - List.of(Map.of("apiName", "profit", "dataType", "Number"), Map.of("apiName", "units", "dataType", "Number"))))); - return new MetricCompilationPlan(source, new MetricFieldResolver(source, target)); - } - @Test - void dependenciesKeepAggregateLevelAndInferredDatatypeAndAreCached() { - var plan = plan(List.of(metric("ratio", "total / count_units"), metric("total", "SUM(orders.profit)"), - metric("count_units", "COUNT(orders.units)"))); - var result = plan.compile("ratio"); - assertEquals(ExpressionCompiler.Level.AGGREGATE, result.level()); - assertEquals("Decimal", result.datatype()); - assertEquals(java.util.Set.of("orders"), result.datasets()); - assertEquals("Integer", plan.compile("count_units").datatype()); - assertSame(result, plan.compile("ratio")); - } - @Test - void bracketedTableauMetricReferencesUseTheSameDependencyChecks() { - var plan = plan(List.of(metric("result", "[total] + 1", "TABLEAU"), metric("total", "SUM(orders.profit)"))); - assertEquals("((SUM([orders].[profit])) + 1)", plan.compile("result").expression()); - } - @Test - void forwardReferencesAndNormalizedSqlNamesAreResolved() { - var plan = plan(List.of(metric("result", "TOTAL + 1"), metric("total", "SUM(orders.profit)"))); - assertTrue(plan.compile("result").expression().contains("SUM([orders].[profit])")); - } - @Test - void ambiguousUnqualifiedFieldOrMetricIsRejected() { - var plan = plan(List.of(metric("result", "profit + 1"), metric("profit", "SUM(orders.profit)"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("result")).getMessage().contains("ambiguous field or metric")); - assertDoesNotThrow(() -> plan.compile("profit")); - } - @Test - void ambiguousCaseFoldedMetricNamesAreRejected() { - var plan = plan(List.of(metric("result", "total + 1"), metric("total", "1"), metric("TOTAL", "2"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("result")).getMessage().contains("ambiguous")); - } - @Test - void reportsDependencyCycleWithPathAndCanStillCompileIndependentMetric() { - var plan = plan(List.of(metric("a", "b + 1"), metric("b", "c + 1"), metric("c", "a + 1"), metric("ok", "1"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("a")).getMessage().contains("a -> b -> c -> a")); - assertEquals("1", plan.compile("ok").expression()); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("a")).getMessage().contains("a -> b -> c -> a")); - } - @Test - void rejectsAggregateOfAnAggregateMetric() { - var plan = plan(List.of(metric("bad", "SUM(total)"), metric("total", "SUM(orders.profit)"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("bad")).getMessage().contains("nested aggregate")); - } - @Test - void rejectsRowAndAggregateMixAcrossDependency() { - var plan = plan(List.of(metric("bad", "total + orders.profit"), metric("total", "SUM(orders.profit)"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("bad")).getMessage().contains("mix aggregate")); - } - @Test - void allNullDependencyRetainsItsExplicitDeclaredType() { - var empty = metric("empty", "NULL"); - empty.put("datatype", "Integer"); - var plan = plan(List.of(metric("result", "COALESCE(empty, 1)"), empty)); - assertEquals("Integer", plan.compile("result").datatype()); - } - @Test - void fractionalDependencyCannotSatisfyIntegerDeclaration() { - var integer = metric("rounded", "total"); - integer.put("datatype", "Integer"); - var plan = plan(List.of(integer, metric("total", "AVG(orders.units)"))); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("rounded")).getMessage().contains("incompatible")); - } - @Test - void boundsDependencyDepthBeforeStackExhaustion() { - List> metrics = new ArrayList<>(); - for (int i = 0; i < 150; i++) metrics.add(metric("m" + i, i == 149 ? "1" : "m" + (i + 1) + " + 1")); - var plan = plan(metrics); - assertTrue(assertThrows(ConversionException.class, () -> plan.compile("m0")).getMessage().contains("dependency depth")); - } - @Test - void boundsExponentialDependencyExpansion() { - List> metrics = new ArrayList<>(); - metrics.add(metric("m0", "SUM(orders.profit)")); - for (int i = 1; i < 20; i++) metrics.add(metric("m" + i, "m" + (i - 1) + " + m" + (i - 1))); - var plan = plan(metrics); - var error = assertThrows(ConversionException.class, () -> plan.compile("m19")); - assertTrue(error.getMessage().contains("exceeds 131072 characters"), error.getMessage()); - } - @Test - void sharedDependencyAcrossManyMetricsDoesNotAlterTheCompiledResult() { - List> metrics = new ArrayList<>(); - metrics.add(metric("base", "SUM(orders.profit)")); - for (int i = 0; i < 500; i++) metrics.add(metric("m" + i, "base + " + i)); - var plan = plan(metrics); - for (int i = 0; i < 500; i++) assertEquals("((SUM([orders].[profit])) + " + i + ")", plan.compile("m" + i).expression()); - assertSame(plan.compile("base"), plan.compile("base")); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java index ee8407c3..e6a09ba0 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java @@ -22,12 +22,7 @@ import static org.junit.jupiter.api.Assertions.*; import java.math.BigDecimal; -import java.math.MathContext; import java.math.RoundingMode; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.temporal.ChronoField; -import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -174,43 +169,6 @@ void textPredicatesPreserveDuplicatesNullsAndEscapedApostrophes(String dialect) assertValue(dialect, "COUNT(DISTINCT orders.status)", List.of(row(null, null, null, null)), 0.0); } - @ParameterizedTest - @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) - void dateAndEmailDomainRowExpressionsEvaluateAtBoundaryValues(String dialect) { - Map types = Map.of("date", "Date", "timestamp", "DateTime", "email", "String", "amount", "Decimal"); - Map row = new LinkedHashMap<>(); - row.put("date", LocalDate.of(2024, 2, 29)); - row.put("timestamp", LocalDateTime.of(2025, 1, 1, 0, 0)); - row.put("amount", new BigDecimal("0.1000000000000000000000001")); - assertScalar(dialect, "YEAR(date)", row, types, new BigDecimal("2024")); - assertScalar(dialect, "YEAR(timestamp)", row, types, new BigDecimal("2025")); - row.put("date", null); - assertScalar(dialect, "YEAR(date)", row, types, null); - String domain = "SUBSTRING(email, POSITION('@' IN email) + 1, LENGTH(email))"; - for (String address : List.of("alice@example.com", "no-at-sign", "@", "", "a@b@c")) { - row.put("email", address); - assertScalar(dialect, domain, row, types, address.substring(address.indexOf('@') + 1)); - } - row.put("email", null); - assertScalar(dialect, domain, row, types, null); - assertScalar(dialect, "amount = 0.1000000000000000000000001", row, types, Boolean.TRUE); - assertScalar(dialect, "amount + 0.2", row, types, new BigDecimal("0.3000000000000000000000001")); - assertScalar(dialect, "NULLIF(amount, 0.1000000000000000000000001)", row, types, null); - } - - private static void assertScalar(String dialect, String source, Map row, - Map types, Object expected) { - var compiled = ExpressionCompiler.compile(ExpressionCompiler.parse(source, dialect), reference -> { - String name = reference.parts().getLast().text(); - return new ExpressionCompiler.Binding("[orders].[" + name + "]", types.get(name), "orders"); - }); - Object actual = new TuaSubsetEvaluator(compiled.expression()).evaluateRow(row); - if (expected instanceof BigDecimal number) { - assertInstanceOf(Number.class, actual); - assertEquals(0, number.compareTo(new BigDecimal(actual.toString())), source); - } else assertEquals(expected, actual, source); - } - private static void assertValue( String dialect, String sql, List> rows, Double expected) { Map metric = Map.of( @@ -227,11 +185,11 @@ private static void assertValue( Map target = Map.of("semanticDataObjects", List.of(Map.of( "apiName", "orders", "semanticMeasurements", List.of( - Map.of("apiName", "amount", "dataType", "Number"), - Map.of("apiName", "cost", "dataType", "Number")), + Map.of("apiName", "amount", "dataObjectFieldName", "amount__c", "dataType", "Number"), + Map.of("apiName", "cost", "dataObjectFieldName", "cost__c", "dataType", "Number")), "semanticDimensions", List.of( - Map.of("apiName", "flag", "dataType", "Boolean"), - Map.of("apiName", "status", "dataType", "Text"))))); + Map.of("apiName", "flag", "dataObjectFieldName", "flag__c", "dataType", "Boolean"), + Map.of("apiName", "status", "dataObjectFieldName", "status__c", "dataType", "Text"))))); MetricExpressionTranslator.Result translated = MetricExpressionTranslator.translate(metric, source, target); assertEquals("Number", translated.dataType(), sql); Object actual = new TuaSubsetEvaluator(translated.expression()).evaluate(rows); @@ -286,12 +244,6 @@ Object evaluate(List> rows) { return calculation.value(rows, Map.of()); } - Object evaluateRow(Map row) { - Calculation calculation = expression(0); - assertEquals(tokens.size(), position, "Unconsumed generated Tua tokens"); - return calculation.value(List.of(row), row); - } - private Calculation expression(int minimum) { Calculation left = prefix(); while (position < tokens.size() && precedence(tokens.get(position)) >= minimum) { @@ -323,7 +275,7 @@ private Calculation prefix() { Calculation child = expression(token.equals("-") ? 7 : 3); return (rows, row) -> { Object value = child.value(rows, row); - return value == null ? null : token.equals("-") ? decimal(value).negate() : !(Boolean) value; + return value == null ? null : token.equals("-") ? -number(value) : !(Boolean) value; }; } if (token.equalsIgnoreCase("NULL")) { @@ -348,7 +300,7 @@ private Calculation prefix() { }; } if (Character.isDigit(token.charAt(0))) { - return (rows, row) -> new BigDecimal(token); + return (rows, row) -> Double.valueOf(token); } expect("("); List arguments = new ArrayList<>(); @@ -367,26 +319,24 @@ private static Calculation function(String name, List arguments) { .map(input -> arguments.getFirst().value(rows, input)) .filter(java.util.Objects::nonNull).toList(); if (name.equals("COUNT")) { - return BigDecimal.valueOf(values.size()); + return (double) values.size(); } if (name.equals("COUNTD")) { - return BigDecimal.valueOf(values.stream().map(value -> value instanceof Number - ? decimal(value).stripTrailingZeros() : value).distinct().count()); + return (double) values.stream().distinct().count(); } if (values.isEmpty()) { return null; } return switch (name) { - case "SUM" -> values.stream().map(TuaSubsetEvaluator::decimal).reduce(BigDecimal.ZERO, BigDecimal::add); - case "AVG" -> values.stream().map(TuaSubsetEvaluator::decimal).reduce(BigDecimal.ZERO, BigDecimal::add) - .divide(BigDecimal.valueOf(values.size()), MathContext.DECIMAL128); - case "MIN" -> values.stream().min(TuaSubsetEvaluator::compare).orElseThrow(); - case "MAX" -> values.stream().max(TuaSubsetEvaluator::compare).orElseThrow(); + case "SUM" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).sum(); + case "AVG" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).average().orElseThrow(); + case "MIN" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).min().orElseThrow(); + case "MAX" -> values.stream().mapToDouble(TuaSubsetEvaluator::number).max().orElseThrow(); default -> throw new AssertionError(name); }; }; } - assertTrue(List.of("IFNULL", "ISNULL", "ABS", "CEILING", "FLOOR", "ROUND", "YEAR", "LEN", "FIND", "MID").contains(name), + assertTrue(List.of("IFNULL", "ISNULL", "ABS", "CEILING", "FLOOR", "ROUND").contains(name), "Unsupported generated function: " + name); return (rows, row) -> { Object value = arguments.getFirst().value(rows, row); @@ -401,28 +351,13 @@ private static Calculation function(String name, List arguments) { if (value == null) { return null; } - if (name.equals("FIND")) { - Object needle = arguments.get(1).value(rows, row); - return needle == null ? null : BigDecimal.valueOf(((String) value).indexOf((String) needle) + 1); - } - if (name.equals("MID")) { - Object startValue = arguments.get(1).value(rows, row); - Object lengthValue = arguments.size() > 2 ? arguments.get(2).value(rows, row) : null; - if (startValue == null || arguments.size() > 2 && lengthValue == null) return null; - int start = decimal(startValue).intValueExact() - 1; - String string = (String) value; - if (start >= string.length()) return ""; - int end = arguments.size() > 2 ? Math.min(string.length(), start + decimal(lengthValue).intValueExact()) : string.length(); - return string.substring(start, end); - } return switch (name) { - case "YEAR" -> BigDecimal.valueOf(((TemporalAccessor) value).get(ChronoField.YEAR)); - case "LEN" -> BigDecimal.valueOf(((String) value).length()); - case "ABS" -> decimal(value).abs(); - case "CEILING" -> decimal(value).setScale(0, RoundingMode.CEILING); - case "FLOOR" -> decimal(value).setScale(0, RoundingMode.FLOOR); - case "ROUND" -> decimal(value).setScale( - arguments.size() == 1 ? 0 : decimal(arguments.get(1).value(rows, row)).intValueExact(), RoundingMode.HALF_UP); + case "ABS" -> Math.abs(number(value)); + case "CEILING" -> Math.ceil(number(value)); + case "FLOOR" -> Math.floor(number(value)); + case "ROUND" -> BigDecimal.valueOf(number(value)).setScale( + arguments.size() == 1 ? 0 : (int) number(arguments.get(1).value(rows, row)), + RoundingMode.HALF_UP).doubleValue(); default -> throw new AssertionError("Unsupported generated function: " + name); }; }; @@ -445,31 +380,25 @@ private static Object binary(String operator, Object left, Object right) { return null; } return switch (operator) { - case "+" -> decimal(left).add(decimal(right)); - case "-" -> decimal(left).subtract(decimal(right)); - case "*" -> decimal(left).multiply(decimal(right)); + case "+" -> number(left) + number(right); + case "-" -> number(left) - number(right); + case "*" -> number(left) * number(right); case "/" -> { - assertNotEquals(0, decimal(right).signum(), "Generated expression evaluated an unguarded zero divisor"); - yield decimal(left).divide(decimal(right), MathContext.DECIMAL128); + assertNotEquals(0.0, number(right), "Generated expression evaluated an unguarded zero divisor"); + yield number(left) / number(right); } - case "=" -> compare(left, right) == 0; - case "!=", "<>" -> compare(left, right) != 0; - case "<" -> compare(left, right) < 0; - case "<=" -> compare(left, right) <= 0; - case ">" -> compare(left, right) > 0; - case ">=" -> compare(left, right) >= 0; + case "=" -> left.equals(right); + case "!=", "<>" -> !left.equals(right); + case "<" -> number(left) < number(right); + case "<=" -> number(left) <= number(right); + case ">" -> number(left) > number(right); + case ">=" -> number(left) >= number(right); default -> throw new AssertionError(operator); }; } - private static BigDecimal decimal(Object value) { - return value instanceof BigDecimal decimal ? decimal : new BigDecimal(value.toString()); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static int compare(Object left, Object right) { - if (left instanceof Number && right instanceof Number) return decimal(left).compareTo(decimal(right)); - return ((Comparable) left).compareTo(right); + private static double number(Object value) { + return ((Number) value).doubleValue(); } private static int precedence(String token) { diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java index 9bfc151a..8ca36803 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java @@ -45,7 +45,7 @@ private static Map source() { private static Map target() { return Map.of("semanticDataObjects", List.of(Map.of("apiName", "orders", "semanticMeasurements", - TYPES.entrySet().stream().map(entry -> Map.of("apiName", entry.getKey(), "dataType", + TYPES.entrySet().stream().map(entry -> Map.of("apiName", entry.getKey(), "dataObjectFieldName", entry.getKey() + "__c", "dataType", SalesforceDataTypeMapper.toSalesforce(entry.getValue()))).toList()))); } @@ -160,6 +160,8 @@ static Stream unsupported() { Arguments.of("NULLIF(SUM(orders.amount))", "expects 2"), Arguments.of("MEDIAN(orders.amount)", "unsupported function"), Arguments.of("CAST(orders.amount AS DECIMAL)", "unsupported SQL expression CastExpression"), + Arguments.of("SUM(YEAR(orders.ordered))", "unsupported function"), + Arguments.of("SUM(LENGTH(orders.status))", "unsupported function"), Arguments.of("SUM(orders.amount) OVER ()", "unsupported SQL expression AnalyticExpression"), Arguments.of("SUM(orders.amount) FILTER (WHERE orders.active)", "unsupported SQL expression AnalyticExpression"), Arguments.of("{ FIXED : SUM(orders.amount) }", "unsupported character"), @@ -242,6 +244,8 @@ void outputDatatypeMustAgreeWithTheFormula() { void limitsNestingAndExpansionWithoutStackOverflow() { assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "(".repeat(200) + "1" + ")".repeat(200))); assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "-".repeat(200) + "1")); + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "1+".repeat(5000) + "1")); + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", "1".repeat(32769))); String formula = "SUM(orders.amount)"; for (int i = 0; i < 20; i++) formula = "NULLIF(" + formula + ", 0)"; String expanded = formula; @@ -254,18 +258,72 @@ void temporalAggregatesCanBeUsedInNumericPredicates() { translate("SNOWFLAKE", "CASE WHEN MIN(orders.ordered) = MAX(orders.ordered) THEN 1 ELSE 0 END")); } + @ParameterizedTest + @ValueSource(strings = {"SUM(ALL orders.amount)", "SUM(UNIQUE orders.amount)", + "SUM(orders.amount IGNORE NULLS)", "SUM(orders.amount RESPECT NULLS)", + "SUM(orders.amount) IGNORE NULLS", "SUM(orders.amount LIMIT 1)", + "SUM(orders.amount HAVING MAX orders.quantity)", "SUM(orders.amount ORDER BY orders.quantity)", + "SUM(orders.amount) KEEP (DENSE_RANK LAST ORDER BY orders.quantity)", + "SUM(orders.amount).attribute", "private_schema.SUM(orders.amount)", "\"SUM\"(orders.amount)", + "SUM(CASE orders.amount WHEN 1 THEN 2 ELSE 0 END)", "N'prefixed'", + "orders.status ISNULL", "orders.status NOTNULL", "PRIOR orders.amount = orders.quantity", + "!orders.active", "(SELECT amount FROM orders)", "orders.amount IN (1, 2)", + "SUM(orders.amount) AS alias", "orders.amount(+) = orders.quantity"}) + void parserAcceptanceNeverDiscardsUnsupportedSqlModifiers(String expression) { + assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", expression), expression); + } + + @ParameterizedTest + @ValueSource(strings = {"CEIL(AVG(orders.amount))", "FLOOR(AVG(orders.amount))", + "ROUND(AVG(orders.amount))", "ROUND(AVG(orders.amount), 0)", "ROUND(AVG(orders.amount), -2)"}) + void integralRoundingSatisfiesAnIntegerMetricDeclaration(String expression) { + Map metric = metric("SNOWFLAKE", expression); + metric.put("datatype", "Integer"); + String output = MetricExpressionTranslator.translate(metric, source(), target()).expression(); + metric.put("expression", Map.of("dialects", List.of(Map.of("dialect", "TABLEAU", "expression", output)))); + assertEquals(output, MetricExpressionTranslator.translate(metric, source(), target()).expression()); + } + + @Test + void positiveRoundingPrecisionDoesNotClaimAnIntegralResult() { + Map metric = metric("SNOWFLAKE", "ROUND(AVG(orders.amount), 2)"); + metric.put("datatype", "Integer"); + assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, source(), target())); + assertEquals("0.12345678901234567890123456789", translate("SNOWFLAKE", "0.12345678901234567890123456789")); + } + + @Test + void quotedDotsAndEscapesPreserveFieldReferenceBoundaries() { + Map source = Map.of("datasets", List.of(Map.of("name", "ORDER.ITEMS", "fields", + List.of(Map.of("name", "NET REVENUE", "datatype", "Decimal"))))); + Map target = Map.of("semanticDataObjects", List.of(Map.of("apiName", "ORDER.ITEMS", + "semanticMeasurements", List.of(Map.of("apiName", "NET REVENUE", "dataType", "Number", + "dataObjectFieldName", "net__c"))))); + for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { + assertEquals("SUM([ORDER.ITEMS].[NET REVENUE])", MetricExpressionTranslator.translate( + metric(dialect, "SUM(\"ORDER.ITEMS\".\"NET REVENUE\")"), source, target).expression()); + var field = (MetricExpression.Field) SqlMetricExpressionParser.parse("\"a\"\"b.c\".\"d\"\"e\"", dialect); + assertEquals(List.of(new MetricFieldResolver.Identifier("a\"b.c", true), + new MetricFieldResolver.Identifier("d\"e", true)), field.parts()); + } + assertThrows(IllegalArgumentException.class, () -> SqlMetricExpressionParser.parse("[a.b].[c]]d]", "ANSI_SQL")); + } + @Test void separateAggregatesRequireConnectedDatasets() { - Map twoSources = Map.of("datasets", List.of( + Map twoSources = Map.of("relationships", List.of(Map.of("name", "orders_returns", + "from", "orders", "to", "returns", "from_columns", List.of("amount"), "to_columns", List.of("amount"))), + "datasets", List.of( Map.of("name", "orders", "fields", List.of(Map.of("name", "amount", "datatype", "Decimal"))), Map.of("name", "returns", "fields", List.of(Map.of("name", "amount", "datatype", "Decimal"))))); Map twoTargets = new LinkedHashMap<>(Map.of("semanticDataObjects", List.of( - Map.of("apiName", "orders", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number"))), - Map.of("apiName", "returns", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataType", "Number")))))); + Map.of("apiName", "orders", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c", "dataType", "Number"))), + Map.of("apiName", "returns", "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c", "dataType", "Number")))))); Map metric = metric("SNOWFLAKE", "SUM(orders.amount) - SUM(returns.amount)"); assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate(metric, twoSources, twoTargets)); - twoTargets.put("semanticRelationships", List.of(Map.of("leftSemanticDefinitionApiName", "orders", - "rightSemanticDefinitionApiName", "returns"))); + twoTargets.put("semanticRelationships", List.of(Map.of("apiName", "orders_returns", "isEnabled", true, + "leftSemanticDefinitionApiName", "orders", "rightSemanticDefinitionApiName", "returns", "criteria", + List.of(Map.of("leftSemanticFieldApiName", "amount", "rightSemanticFieldApiName", "amount"))))); assertEquals("(SUM([orders].[amount]) - SUM([returns].[amount]))", MetricExpressionTranslator.translate(metric, twoSources, twoTargets).expression()); assertThrows(ConversionException.class, () -> MetricExpressionTranslator.translate( diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java index 2f27d585..bb9d64fc 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricFieldResolverTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import org.apache.ossie.converter.MetricFieldResolver.Identifier; @@ -92,6 +93,7 @@ void resolvesQuotedDeclarationsWithoutRenamingExportedObjects() { void tableauReferencesUseExactApiNamesAndRequireDataset() { MetricFieldResolver resolver = resolver("Orders", "revenue", "Integer", "Number"); assertEquals("[Orders].[revenue]", resolver.resolve(sql("Orders", "revenue"), true).expression()); + assertEquals("[Orders].[revenue]", resolver.resolve(sql("orders", "revenue"), false).expression()); assertError(resolver, sql("orders", "revenue"), true, "Unknown dataset"); assertError(resolver, sql("Orders", "Revenue"), true, "Unknown field"); assertError(resolver, sql("revenue"), true, "must use [dataset].[field]"); @@ -146,6 +148,19 @@ void checksThatDeclaredFieldsWereActuallyExportedAsDirectFields() { assertError(calculatedField, sql("Orders", "amount"), false, "calculated or omitted fields"); } + @ParameterizedTest + @ValueSource(strings = {"profit+tax", "profit-tax", "profit=tax", "1", "TRUE", "(profit)", + "Orders.profit", "\"profit\"", "[profit]", "SUM(profit)", "profit;tax", ""}) + void rejectsExpressionsMisclassifiedAsPhysicalFields(String binding) { + Map target = new LinkedHashMap<>(targetField("revenue", "Number")); + if (binding.isEmpty()) target.remove("dataObjectFieldName"); + else target.put("dataObjectFieldName", binding); + MetricFieldResolver resolver = new MetricFieldResolver( + Map.of("datasets", List.of(dataset("Orders", field("revenue", "Decimal")))), + Map.of("semanticDataObjects", List.of(targetDataset("Orders", target)))); + assertError(resolver, sql("Orders", "revenue"), false, "revenue"); + } + @Test void rejectsDuplicateExportedObjectsAndFieldsAcrossKinds() { Map source = Map.of("datasets", List.of(dataset("Orders", field("amount", "Decimal")))); @@ -196,21 +211,92 @@ private static MetricFieldResolver resolver(String dataset, String field, String @Test void disabledRelationshipsDoNotConnectDatasets() { - MetricFieldResolver resolver = graphResolver(List.of(Map.of( - "leftSemanticDefinitionApiName", "Orders", - "rightSemanticDefinitionApiName", "Returns", "isEnabled", false)), "Orders", "Returns"); + Map disabled = new LinkedHashMap<>(relationship("Orders", "Returns")); + disabled.put("isEnabled", false); + MetricFieldResolver resolver = graphResolver(List.of(disabled), "Orders", "Returns"); assertThrows(IllegalArgumentException.class, () -> resolver.validateDatasets(java.util.Set.of("Orders", "Returns"))); } + @ParameterizedTest + @ValueSource(strings = {"changed endpoint", "swapped keys", "empty criteria", "missing criterion", + "missing join field", "formula join", "missing enabled", "missing declaration", "duplicate declaration"}) + void unverifiedRelationshipsCannotEstablishConnectivity(String defect) { + Map valid = relationship("Orders", "Customers"); + Map changed = new LinkedHashMap<>(valid); + List> declarations = List.of(sourceRelationship(valid)); + switch (defect) { + case "changed endpoint" -> changed.put("rightSemanticDefinitionApiName", "Returns"); + case "swapped keys" -> changed.put("criteria", List.of(criterion("id", "tenant"), criterion("tenant", "id"))); + case "empty criteria" -> changed.put("criteria", List.of()); + case "missing criterion" -> changed.put("criteria", List.of(criterion("id", "id"))); + case "missing join field" -> { + changed.put("criteria", List.of(criterion("missing", "id"), criterion("tenant", "tenant"))); + Map source = new LinkedHashMap<>(sourceRelationship(valid)); + source.put("from_columns", List.of("missing", "tenant")); + declarations = List.of(source); + } + case "formula join" -> changed.put("criteria", List.of( + Map.of("leftSemanticFieldApiName", "id", "rightSemanticFieldApiName", "id", "leftFieldType", "Formula"), + criterion("tenant", "tenant"))); + case "missing enabled" -> changed.remove("isEnabled"); + case "missing declaration" -> declarations = List.of(); + case "duplicate declaration" -> declarations = List.of(sourceRelationship(valid), sourceRelationship(valid)); + default -> throw new AssertionError(defect); + } + MetricFieldResolver resolver = graphResolver(declarations, List.of(changed), "Orders", "Customers", "Returns"); + Set referenced = Set.of("Orders", defect.equals("changed endpoint") ? "Returns" : "Customers"); + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> resolver.validateDatasets(referenced), defect); + assertTrue(error.getMessage().contains("disconnected"), error.getMessage()); + } + + @Test + void computedPhysicalJoinKeyCannotEstablishConnectivity() { + Map edge = relationship("Orders", "Customers"); + Map computed = new LinkedHashMap<>(targetField("id", "Number")); + computed.put("dataObjectFieldName", "profit+tax"); + MetricFieldResolver resolver = new MetricFieldResolver(Map.of( + "datasets", List.of(dataset("Orders", field("id", "Integer"), field("tenant", "Integer")), + dataset("Customers", field("id", "Integer"), field("tenant", "Integer"))), + "relationships", List.of(sourceRelationship(edge))), Map.of( + "semanticDataObjects", List.of(targetDataset("Orders", computed, targetField("tenant", "Number")), + targetDataset("Customers", targetField("id", "Number"), targetField("tenant", "Number"))), + "semanticRelationships", List.of(edge))); + assertThrows(IllegalArgumentException.class, + () -> resolver.validateDatasets(Set.of("Orders", "Customers"))); + } + private static MetricFieldResolver graphResolver(List> relationships, String... datasets) { - return new MetricFieldResolver(Map.of(), Map.of( - "semanticDataObjects", java.util.Arrays.stream(datasets).map(name -> targetDataset(name)).toList(), + return graphResolver(relationships.stream().map(MetricFieldResolverTest::sourceRelationship).toList(), + relationships, datasets); + } + + private static MetricFieldResolver graphResolver(List> declarations, + List> relationships, String... datasets) { + return new MetricFieldResolver(Map.of( + "datasets", java.util.Arrays.stream(datasets) + .map(name -> dataset(name, field("id", "Integer"), field("tenant", "Integer"))).toList(), + "relationships", declarations), Map.of( + "semanticDataObjects", java.util.Arrays.stream(datasets) + .map(name -> targetDataset(name, targetField("id", "Number"), targetField("tenant", "Number"))).toList(), "semanticRelationships", relationships)); } private static Map relationship(String left, String right) { - return Map.of("leftSemanticDefinitionApiName", left, "rightSemanticDefinitionApiName", right); + return Map.of("apiName", left + "_to_" + right, "leftSemanticDefinitionApiName", left, + "rightSemanticDefinitionApiName", right, "isEnabled", true, + "criteria", List.of(criterion("id", "id"), criterion("tenant", "tenant"))); + } + + private static Map criterion(String left, String right) { + return Map.of("leftSemanticFieldApiName", left, "rightSemanticFieldApiName", right); + } + + private static Map sourceRelationship(Map target) { + return Map.of("name", target.get("apiName"), "from", target.get("leftSemanticDefinitionApiName"), + "to", target.get("rightSemanticDefinitionApiName"), + "from_columns", List.of("id", "tenant"), "to_columns", List.of("id", "tenant")); } private static List sql(String... parts) { @@ -223,7 +309,8 @@ private static Map dataset(String name, Map... f } private static Map field(String name, String datatype) { - return Map.of("name", name, "datatype", datatype); + return Map.of("name", name, "datatype", datatype, "expression", Map.of("dialects", List.of( + Map.of("dialect", "ANSI_SQL", "expression", "physical_column__c")))); } @SafeVarargs @@ -232,7 +319,7 @@ private static Map targetDataset(String name, Map targetField(String name, String datatype) { - return Map.of("apiName", name, "dataType", datatype); + return Map.of("apiName", name, "dataType", datatype, "dataObjectFieldName", "physical_column__c"); } private static void assertError(MetricFieldResolver resolver, List parts, diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java deleted file mode 100644 index eaf8ab3e..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/RelationshipMappingHandlerTest.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class RelationshipMappingHandlerTest { - private final CustomExtensionHandler extensions = new CustomExtensionHandler(new ObjectMapper()); - private final RelationshipMappingHandler handler = - new RelationshipMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, extensions); - - @Test - void invalidFirstRelationshipFailsWithoutMisassigningItsKeysToTheSecond() { - Map bad = relation("bad", "computed"); - Map good = relation("good", "customer_id"); - Map source = source(bad, good); - Map target = target(); - Map mappings = mappings(); - ConversionException error = assertThrows(ConversionException.class, - () -> handler.execute(source, target, mappings)); - assertTrue(error.getMessage().contains("bad")); - assertTrue(error.getMessage().contains("calculated join keys")); - assertEquals(List.of(bad, good), source.get("relationships"), "Do not filter or mutate source relationships"); - assertFalse(target.containsKey("semanticRelationships"), "No partially mapped valid relationship is published"); - assertFalse(mappings.containsKey("relationships")); - } - - @Test - void allInvalidRelationshipsCannotFallThroughToGenericRawMapping() { - Map source = source(relation("bad", "computed")); - Map target = target(); - Map mappings = mappings(); - assertThrows(ConversionException.class, () -> handler.execute(source, target, mappings)); - new SemanticModelMappingHandler(ConversionDirection.OSSIE_TO_SALESFORCE, extensions) - .execute(source, target, mappings); - assertFalse(target.containsKey("semanticRelationships")); - } - - @Test - void mapsEveryCompositeKeyPairAndUsesOssieDirectionForDefaultCardinality() { - Map relation = relation("join", "customer_id"); - relation.put("from_columns", List.of("customer_id", "region")); - relation.put("to_columns", List.of("id", "region")); - Map target = target(); - handler.execute(source(relation), target, mappings()); - Map exported = relationships(target).get(0); - assertEquals("ManyToOne", exported.get("cardinality")); - assertEquals(true, exported.get("isEnabled")); - assertEquals("Auto", exported.get("joinType")); - assertEquals(List.of(Map.of("leftSemanticFieldApiName", "customer_id", "rightSemanticFieldApiName", "id"), - Map.of("leftSemanticFieldApiName", "region", "rightSemanticFieldApiName", "region")), exported.get("criteria")); - } - - @ParameterizedTest - @ValueSource(strings = {"OneToOne", "OneToMany", "ManyToOne", "ManyToMany", "Unspecified"}) - void preservesExplicitNativeCardinalityForSalesforceRoundTrips(String cardinality) { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"cardinality\":\"" + cardinality + "\",\"isEnabled\":false,\"joinType\":\"Left\"}"))); - Map target = target(); - handler.execute(source(relation), target, mappings()); - Map exported = relationships(target).get(0); - assertEquals(cardinality, exported.get("cardinality")); - assertEquals(false, exported.get("isEnabled")); - assertEquals("Left", exported.get("joinType")); - } - - @Test - void rejectsInvalidNativeCardinalityInsteadOfSilentlyReplacingIt() { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"cardinality\":\"Guess\"}"))); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("cardinality")); - } - - @ParameterizedTest - @ValueSource(strings = {"cardinality", "isEnabled", "joinType"}) - void doesNotReplaceExplicitNullNativeMetadataWithDefaults(String property) { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"" + property + "\":null}"))); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source(relation), target(), mappings())).getMessage().contains(property)); - } - - @Test - void rejectsCompositeArityMismatch() { - Map relation = relation("join", "customer_id"); - relation.put("from_columns", List.of("customer_id", "region")); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("same number")); - } - - @ParameterizedTest - @ValueSource(strings = {"from", "to"}) - void rejectsUnknownEndpoint(String endpoint) { - Map relation = relation("join", "customer_id"); - relation.put(endpoint, "missing"); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("missing")); - } - - @Test - void rejectsDuplicateRelationNamesAndRepeatedKeys() { - assertTrue(assertThrows(ConversionException.class, () -> handler.execute( - source(relation("join", "customer_id"), relation("JOIN", "customer_id")), target(), mappings())) - .getMessage().contains("Duplicate")); - Map relation = relation("join", "customer_id"); - relation.put("from_columns", List.of("customer_id", "customer_id")); - relation.put("to_columns", List.of("id", "region")); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source(relation), target(), mappings())).getMessage().contains("duplicate field")); - } - - @Test - void declaredUniquenessMustCoverTheRelationshipTargetKey() { - Map source = source(relation("join", "customer_id")); - datasets(source).get(1).put("primary_key", List.of("region")); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source, target(), mappings())).getMessage().contains("declared primary_key")); - datasets(source).get(1).put("unique_keys", List.of(List.of("id"))); - Map target = target(); - assertDoesNotThrow(() -> handler.execute(source, target, mappings())); - assertFalse(relationships(target).get(0).containsKey("primaryNameField")); - } - - @Test - void nativeOneToManyChecksLeftUniquenessAndAllowsNonUniqueRightKey() { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"cardinality\":\"OneToMany\"}"))); - Map source = source(relation); - datasets(source).get(0).put("primary_key", List.of("customer_id")); - datasets(source).get(1).put("primary_key", List.of("region")); - assertDoesNotThrow(() -> handler.execute(source, target(), mappings())); - datasets(source).get(0).put("primary_key", List.of("region")); - assertTrue(assertThrows(ConversionException.class, - () -> handler.execute(source, target(), mappings())).getMessage().contains("from_columns")); - } - - @ParameterizedTest - @ValueSource(strings = {"ManyToMany", "Unspecified"}) - void nativeNonUniqueCardinalitiesDoNotInventKeyConstraints(String cardinality) { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"cardinality\":\"" + cardinality + "\"}"))); - Map source = source(relation); - datasets(source).get(0).put("primary_key", List.of("region")); - datasets(source).get(1).put("primary_key", List.of("region")); - assertDoesNotThrow(() -> handler.execute(source, target(), mappings())); - } - - @Test - void coreJoinKeysRemainAuthoritativeOverStaleNativeExtensions() { - Map relation = relation("join", "customer_id"); - relation.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"criteria\":[{\"leftSemanticFieldApiName\":\"stale\",\"rightSemanticFieldApiName\":\"stale\"}]}"))); - Map target = target(); - handler.execute(source(relation), target, mappings()); - assertEquals(List.of(Map.of("leftSemanticFieldApiName", "customer_id", "rightSemanticFieldApiName", "id")), - relationships(target).get(0).get("criteria")); - } - - @Test - void salesforceToOssieKeepsNativeCardinalityAndPairOrder() { - Map sf = target(); - handler.execute(source(relation("join", "customer_id")), sf, mappings()); - relationships(sf).get(0).put("cardinality", "OneToMany"); - Map osi = new LinkedHashMap<>(); - Map reverseMappings = new LinkedHashMap<>(); - reverseMappings.put("semanticRelationships", "relationships"); - reverseMappings.put("semanticRelationships.apiName", "relationships.name"); - new RelationshipMappingHandler(ConversionDirection.SALESFORCE_TO_OSSIE, extensions) - .execute(sf, osi, reverseMappings); - @SuppressWarnings("unchecked") - Map relation = ((List>) osi.get("relationships")).get(0); - assertEquals("orders", relation.get("from")); - assertEquals(List.of("customer_id"), relation.get("from_columns")); - assertEquals(List.of("id"), relation.get("to_columns")); - assertTrue(relation.get("custom_extensions").toString().contains("OneToMany")); - } - - private static Map source(Map... relationships) { - Map source = new LinkedHashMap<>(); - source.put("name", "sales"); - source.put("datasets", new ArrayList<>(List.of( - sourceDataset("orders", "customer_id", "region", "computed"), sourceDataset("customers", "id", "region")))); - source.put("relationships", new ArrayList<>(List.of(relationships))); - return source; - } - - private static Map sourceDataset(String name, String... fields) { - Map result = new LinkedHashMap<>(); - result.put("name", name); - result.put("fields", java.util.Arrays.stream(fields).map(field -> Map.of("name", field)).toList()); - return result; - } - - private static Map target() { - Map target = new LinkedHashMap<>(); - target.put("semanticDataObjects", List.of( - Map.of("apiName", "orders", "semanticDimensions", List.of(Map.of("apiName", "customer_id"), Map.of("apiName", "region"))), - Map.of("apiName", "customers", "semanticDimensions", List.of(Map.of("apiName", "id"), Map.of("apiName", "region"))))); - return target; - } - - private static Map relation(String name, String field) { - return new LinkedHashMap<>(Map.of("name", name, "from", "orders", "to", "customers", - "from_columns", List.of(field), "to_columns", List.of("id"))); - } - - private static Map mappings() { - Map mappings = new LinkedHashMap<>(); - mappings.put("name", "apiName"); - mappings.put("relationships", "semanticRelationships"); - mappings.put("relationships.name", "semanticRelationships.apiName"); - return mappings; - } - - @SuppressWarnings("unchecked") - private static List> relationships(Map target) { - return (List>) target.get("semanticRelationships"); - } - - @SuppressWarnings("unchecked") - private static List> datasets(Map source) { - return (List>) source.get("datasets"); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java deleted file mode 100644 index 593406b3..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceBindingsTest.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.apache.ossie.exception.InvalidInputException; -import org.apache.ossie.exception.ValidationException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class SalesforceBindingsTest { - private static final ObjectMapper JSON = new ObjectMapper(); - private static final ObjectMapper YAML = new ObjectMapper(new YAMLFactory()); - private static final String MODEL = "Customer_Orders_Model"; - @TempDir Path directory; - - @Test - void acceptsEmptyCatalogAndLoadsJsonOrYamlFromAFile() throws Exception { - assertTrue(SalesforceBindings.none().isEmpty()); - assertTrue(SalesforceBindings.fromString("models: {}").isEmpty()); - Path file = directory.resolve("bindings.json"); - Files.writeString(file, "{\"models\":{\"sales\":{\"dataspace\":\"production\"}}}"); - SalesforceBindings bindings = SalesforceBindings.fromPath(file); - Map target = new LinkedHashMap<>(); - bindings.apply(Map.of("name", "sales"), target); - assertEquals("production", target.get("dataspace")); - assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromPath(directory.resolve("missing.yaml"))); - } - - @ParameterizedTest - @ValueSource(strings = { - "models: {}\nmodels: {}", - "models: {sales: {}, sales: {}}", - "models: {sales: {dataspace: a, dataspace: b}}", - "models: {sales: {datasets: {orders: {}, orders: {}}}}", - "models: {sales: {datasets: {orders: {fields: {amount: a, amount: b}}}}}" - }) - void rejectsDuplicateKeysAtEveryBindingsLevel(String input) { - InvalidInputException error = assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); - assertTrue(error.getMessage().contains("Duplicate field"), error.getMessage()); - } - - @ParameterizedTest - @ValueSource(strings = { - "models: {}\nunknown: true", - "models: {sales: {metrics: {}}}", - "models: {sales: {expression: 'SUM(amount)'}}", - "models: {sales: {datasets: {orders: {apiName: changed}}}}", - "models: {sales: {datasets: {orders: {expression: 'amount + 1'}}}}", - "models: {sales: {datasets: {orders: {syntax: Tua}}}}" - }) - void rejectsUnknownPropertiesIncludingFormulaAndSemanticNameOverrides(String input) { - InvalidInputException error = assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); - assertTrue(error.getMessage().contains("unknown property"), error.getMessage()); - } - - @ParameterizedTest - @ValueSource(strings = { - "models: {}\n---\nmodels: {sales: {dataspace: hidden}}", - "models: {}\n---\nnull", - "{\"models\":{}}\n{\"models\":{}}" - }) - void rejectsTrailingYamlDocumentsAndJsonValues(String input) { - assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); - } - - @ParameterizedTest - @ValueSource(strings = { - "", "null", "[]", "{}", "models: null", "models: []", "models: {sales: null}", - "models: {'': {}}", "models: {sales: {datasets: null}}", - "models: {sales: {dataspace: 5}}", "models: {sales: {dataspace: ''}}", - "models: {sales: {datasets: {orders: {fields: {amount: {expression: 'amount + 1'}}}}}}", - "models: {sales: {datasets: {orders: {fields: {amount: null}}}}}", - "models: {sales: {datasets: {orders: {fields: {amount: 17}}}}}", - "models: {sales: {datasets: {orders: {dataObjectName: ' '}}}}", - "{\"models\":{\"sales\":{\"dataspace\":\"bad\\nname\"}}}" - }) - void rejectsInvalidShapesAndNonStringOrBlankBindingValues(String input) { - assertThrows(InvalidInputException.class, () -> SalesforceBindings.fromString(input)); - } - - @Test - void absentModelBindingLeavesTargetUntouchedAndNamesMatchExactly() { - SalesforceBindings bindings = SalesforceBindings.fromString("models: {sales: {dataspace: production}}"); - Map target = new LinkedHashMap<>(Map.of("dataspace", "original")); - bindings.apply(Map.of("name", "Sales"), target); - assertEquals(Map.of("dataspace", "original"), target); - assertThrows(ConversionException.class, () -> bindings.validateModels(java.util.Set.of("Sales"))); - } - - @Test - void appliesPhysicalBindingsWithoutMutatingCanonicalModelOrFormulaMetadata() throws Exception { - String fixture = fixture(); - Map original = YAML.readValue(fixture, new TypeReference<>() {}); - Map baseline = output(SalesforceBindings.none(), fixture); - SalesforceBindings bindings = SalesforceBindings.fromString(""" - models: - Customer_Orders_Model: - dataspace: production - datasets: - Orders: - dataObjectName: OrdersProduction__dll - dataObjectType: Dlo - fields: - amount: NetRevenue__c - order_id: OrderIdentifier__c - """); - Map bound = output(bindings, fixture); - assertEquals("production", bound.get("dataspace")); - Map orders = find(items(bound, "semanticDataObjects"), "Orders"); - assertEquals("OrdersProduction__dll", orders.get("dataObjectName")); - assertEquals("NetRevenue__c", find(items(orders, "semanticMeasurements"), "amount").get("dataObjectFieldName")); - assertEquals("OrderIdentifier__c", find(items(orders, "semanticDimensions"), "order_id").get("dataObjectFieldName")); - assertEquals(baseline.get("semanticCalculatedMeasurements"), bound.get("semanticCalculatedMeasurements")); - assertEquals(baseline.get("semanticCalculatedDimensions"), bound.get("semanticCalculatedDimensions")); - assertEquals(baseline.get("semanticRelationships"), bound.get("semanticRelationships")); - assertEquals(original, YAML.readValue(fixture, new TypeReference>() {})); - // Exercise apply directly against the original map, in addition to the string API. - @SuppressWarnings("unchecked") Map model = (Map) ((List) original.get("semantic_model")).get(0); - String before = JSON.writeValueAsString(original); - bindings.apply(model, baseline); - assertEquals(before, JSON.writeValueAsString(original)); - assertEquals(bound, baseline); - assertEquals(bound, output(bindings, fixture), "bindings must be reusable without conversion state"); - } - - @Test - void rejectsUnknownModelDatasetAndFieldIdentitiesThroughPublicConverter() throws Exception { - String fixture = fixture(); - assertConversionError("unknown model 'missing'", "models: {missing: {dataspace: prod}}", fixture); - assertConversionError("dataset 'Missing'", "models: {" + MODEL + ": {datasets: {Missing: {dataObjectName: x}}}}", fixture); - assertConversionError("field 'missing'", "models: {" + MODEL + ": {datasets: {Orders: {fields: {missing: x}}}}}", fixture); - assertConversionError("dataset 'orders'", "models: {" + MODEL + ": {datasets: {orders: {dataObjectName: x}}}}", fixture); - } - - @Test - void rejectsCalculatedFieldPhysicalMappingsThroughPublicConverter() throws Exception { - assertConversionError("only direct physical fields can be rebound", - "models: {" + MODEL + ": {datasets: {Orders: {fields: {order_year: CalendarYear__c}}}}}", fixture()); - } - - @Test - void nativeSchemaRejectsUnsupportedDataObjectTypeAfterApplyingBindings() throws Exception { - SalesforceBindings bindings = SalesforceBindings.fromString( - "models: {" + MODEL + ": {datasets: {Orders: {dataObjectType: NotANativeObjectType}}}}"); - String fixture = fixture(); - ValidationException error = assertThrows(ValidationException.class, () -> output(bindings, fixture)); - assertTrue(error.getMessage().contains("dataObjectType"), error.getMessage()); - } - - @Test - void oneBindingCatalogCanTargetMultipleModelsWithoutCrossModelLeakage() throws Exception { - Map document = YAML.readValue(fixture(), new TypeReference<>() {}); - @SuppressWarnings("unchecked") Map first = (Map) ((List) document.get("semantic_model")).get(0); - Map second = JSON.convertValue(first, new TypeReference<>() {}); - second.put("name", "Other_Model"); - document.put("semantic_model", List.of(first, second)); - SalesforceBindings bindings = SalesforceBindings.fromString(""" - models: - Customer_Orders_Model: {dataspace: primary} - Other_Model: {dataspace: secondary} - """); - List result = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings) - .convert(YAML.writeValueAsString(document)); - assertEquals("primary", JSON.readTree(result.get(0)).get("dataspace").asText()); - assertEquals("secondary", JSON.readTree(result.get(1)).get("dataspace").asText()); - } - - private static void assertConversionError(String message, String bindings, String fixture) { - ConversionException error = assertThrows(ConversionException.class, - () -> output(SalesforceBindings.fromString(bindings), fixture)); - assertTrue(error.getMessage().contains(message), error.getMessage()); - } - - private static Map output(SalesforceBindings bindings, String fixture) throws Exception { - String json = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE, bindings).convert(fixture).get(0); - return JSON.readValue(json, new TypeReference<>() {}); - } - - private static String fixture() throws Exception { - return Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")); - } - - @SuppressWarnings("unchecked") - private static List> items(Map parent, String key) { - return (List>) parent.getOrDefault(key, List.of()); - } - - private static Map find(List> items, String name) { - return items.stream().filter(item -> name.equals(item.get("apiName"))).findFirst().orElseThrow(); - } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java deleted file mode 100644 index d512adbc..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceModelValidatorTest.java +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.apache.ossie.exception.ConversionException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class SalesforceModelValidatorTest { - private final SalesforceModelValidator validator = new SalesforceModelValidator(); - - @Test - void acceptsCompleteDirectModelWithoutMutatingEitherSide() { - Map source = source(); - Map target = target(); - String originalSource = source.toString(); - String originalTarget = target.toString(); - assertDoesNotThrow(() -> validator.validate(source, target)); - assertEquals(originalSource, source.toString()); - assertEquals(originalTarget, target.toString()); - } - - @ParameterizedTest - @ValueSource(strings = {"semanticDataObjects", "semanticCalculatedMeasurements"}) - void rejectsMissingExportedEntities(String property) { - Map target = target(); - target.remove(property); - assertTrue(error(source(), target).contains("was not exported")); - } - - @Test - void rejectsMissingFieldAndRequiredPhysicalBinding() { - Map target = target(); - object(target).put("semanticMeasurements", List.of()); - assertTrue(error(source(), target).contains("orders.amount")); - target = target(); - field(target).remove("dataObjectFieldName"); - assertTrue(error(source(), target).contains("dataObjectFieldName")); - } - - @Test - void rejectsEquivalentSourceIdentifiersAndDuplicateTargetFields() { - Map source = source(); - sourceObject(source).put("fields", List.of(Map.of("name", "amount"), Map.of("name", "AMOUNT"))); - assertTrue(error(source, target()).contains("Duplicate")); - Map target = target(); - object(target).put("semanticDimensions", List.of(new LinkedHashMap<>(field(target)))); - assertTrue(error(source(), target).contains("Duplicate")); - } - - @Test - void rejectsModelLevelCalculatedNameCollisionAcrossDimensionsAndMetrics() { - Map target = target(); - target.put("semanticCalculatedDimensions", List.of(calculation("total", "1"))); - assertTrue(error(source(), target).contains("Duplicate model-level calculated field")); - } - - @ParameterizedTest - @ValueSource(strings = {"SUM([missing].[amount])", "SUM([orders].[missing])", "[missing]"}) - void rejectsDanglingFinalFormulaReferences(String expression) { - Map target = target(); - calculation(target).put("expression", expression); - String error = error(source(), target); - assertTrue(error.contains("references"), error); - assertTrue(error.contains("missing"), error); - } - - @ParameterizedTest - @ValueSource(strings = {"IF '[missing].[field]' = '[missing].[field]' THEN SUM([orders].[amount]) ELSE 0 END", - "IF \"[missing]\" = \"[missing]\" THEN SUM([orders].[amount]) ELSE 0 END", - "IF 'it''s [missing]' = 'it''s [missing]' THEN SUM([orders].[amount]) ELSE 0 END"}) - void bracketTextInsideStringsDoesNotBecomeAReference(String expression) { - Map target = target(); - calculation(target).put("expression", expression); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @ParameterizedTest - @ValueSource(strings = {"TABLEAU", "SQL", ""}) - void requiresTuaOnFinalCalculatedFields(String syntax) { - Map target = target(); - calculation(target).put("syntax", syntax); - assertTrue(error(source(), target).contains("requires syntax 'Tua'")); - } - - @Test - void detectsCyclesInNativeCalculatedReferences() { - Map target = target(); - calculation(target).put("expression", "[other]"); - target.put("semanticCalculatedDimensions", List.of(calculation("other", "[total]"))); - assertTrue(error(source(), target).contains("Cyclic")); - } - - @Test - void rejectsGenericHandlerResidueInsteadOfTreatingItAsNativeRelationship() { - Map target = target(); - target.put("semanticRelationships", List.of(Map.of("name", "raw", "from", "orders", "to", "orders", - "from_columns", List.of("amount"), "to_columns", List.of("amount")))); - assertTrue(error(source(), target).contains("apiName")); - } - - @Test - void finalMetricConnectivityUsesOnlyEnabledRelationships() { - Map source = source(); - Map target = target(); - datasets(source).add(new LinkedHashMap<>(Map.of("name", "returns", "fields", List.of(Map.of("name", "amount"))))); - objects(target).add(new LinkedHashMap<>(Map.of("apiName", "returns", "dataObjectName", "returns__dll", - "semanticMeasurements", List.of(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c"))))); - calculation(target).put("expression", "SUM([orders].[amount]) - SUM([returns].[amount])"); - Map relationship = new LinkedHashMap<>(Map.of("apiName", "join", "leftSemanticDefinitionApiName", "orders", - "rightSemanticDefinitionApiName", "returns", "criteria", List.of(Map.of("leftSemanticFieldApiName", "amount", - "rightSemanticFieldApiName", "amount")), "cardinality", "ManyToMany", "joinType", "Auto", "isEnabled", false)); - target.put("semanticRelationships", List.of(relationship)); - assertTrue(error(source, target).contains("disconnected by enabled relationships")); - relationship.remove("isEnabled"); - assertTrue(error(source, target).contains("disconnected by enabled relationships")); - relationship.put("isEnabled", true); - assertDoesNotThrow(() -> validator.validate(source, target)); - } - - @Test - void acceptsOptionalNativeRelationshipMetadataWithoutAssumingAnEnabledEdge() { - Map source = source(); - source.remove("metrics"); - Map target = target(); - target.remove("semanticCalculatedMeasurements"); - target.put("semanticRelationships", List.of(Map.of("apiName", "native", "leftSemanticDefinitionApiName", "orders", - "rightSemanticDefinitionApiName", "orders", "criteria", List.of(Map.of("leftSemanticFieldApiName", "amount", - "rightSemanticFieldApiName", "amount"))))); - assertDoesNotThrow(() -> validator.validate(source, target)); - } - - @Test - void declaredPrimaryAndUniqueKeysAreCheckedWithoutInventingNativeSchemaFields() { - Map source = source(); - sourceObject(source).put("primary_key", List.of("amount")); - sourceObject(source).put("unique_keys", List.of(List.of("amount"))); - Map target = target(); - assertDoesNotThrow(() -> validator.validate(source, target)); - assertFalse(object(target).containsKey("primaryNameField")); - sourceObject(source).put("primary_key", List.of("missing")); - assertTrue(error(source, target).contains("key references unknown field")); - } - - @Test - void detectsChangedCompositeCorrespondenceAfterLaterExtensionRestoration() { - Map source = source(); - source.put("relationships", List.of(Map.of("name", "join", "from", "orders", "to", "orders", - "from_columns", List.of("amount"), "to_columns", List.of("amount")))); - Map target = target(); - object(target).put("semanticDimensions", List.of(Map.of("apiName", "other", "dataObjectFieldName", "other__c"))); - target.put("semanticRelationships", List.of(Map.of("apiName", "join", "leftSemanticDefinitionApiName", "orders", - "rightSemanticDefinitionApiName", "orders", "criteria", List.of(Map.of("leftSemanticFieldApiName", "other", - "rightSemanticFieldApiName", "amount")), "cardinality", "ManyToOne", "joinType", "Auto", "isEnabled", true))); - assertTrue(error(source, target).contains("changed join key correspondence")); - } - - @Test - void checksDerivedFieldCoverageUsingTheExistingPlan() { - Map source = source(); - sourceObject(source).put("source", "orders__dll"); - sourceObject(source).put("fields", List.of( - Map.of("name", "amount", "datatype", "Decimal", "expression", Map.of("dialects", - List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount__c")))), - Map.of("name", "derived", "datatype", "Decimal", "expression", Map.of("dialects", - List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount + 1")))))); - Map target = target(); - FieldExpressionPlan plan = new FieldExpressionPlan(source, target); - assertTrue(assertThrows(ConversionException.class, () -> validator.validate(source, target, plan)) - .getMessage().contains("orders.derived")); - target.put("semanticCalculatedDimensions", List.of(calculation(plan.calculatedApiName("orders", "derived"), - "[orders].[amount] + 1"))); - assertDoesNotThrow(() -> validator.validate(source, target, plan)); - } - - @Test - void validatesNativeDependencyMetadataAfterExtensionRestoration() { - Map target = target(); - calculation(target).put("dependencies", List.of(Map.of("dependentDefinitionApiName", "orders", - "dependentFieldApiName", "missing"))); - assertTrue(error(source(), target).contains("dependency references missing field")); - calculation(target).put("dependencies", List.of(Map.of("dependentDefinitionApiName", "orders", - "dependentFieldApiName", "amount"))); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @Test - void compilesNativeOnlyCalculationsInsteadOfTrustingMetadata() { - Map target = target(); - target.put("semanticCalculatedDimensions", List.of(calculation("external", "BOGUS(1)"))); - String failure = error(source(), target); - assertTrue(failure.contains("Native calculated field 'external'"), failure); - assertTrue(failure.contains("BOGUS"), failure); - Map external = calculation("external", "1"); - external.put("dataType", "Text"); - target.put("semanticCalculatedDimensions", List.of(external)); - assertTrue(error(source(), target).contains("conflicts with native dataType")); - } - - @Test - void nativeOnlyMixedAggregationCannotBypassTheSharedAnalyzer() { - Map target = target(); - field(target).put("dataType", "Number"); - target.put("semanticCalculatedMeasurements", List.of(calculation(target), - calculation("external", "SUM([orders].[amount]) + [orders].[amount]"))); - String failure = error(source(), target); - assertTrue(failure.contains("Native calculated field 'external'"), failure); - assertTrue(failure.contains("aggregate") || failure.contains("aggregat"), failure); - } - - @Test - void explicitlyMarkedNativeRowMeasurementsRemainSupported() { - Map target = target(); - field(target).put("dataType", "Number"); - Map external = calculation("external", "[orders].[amount] * 2"); - target.put("semanticCalculatedMeasurements", List.of(calculation(target), external)); - assertTrue(error(source(), target).contains("expression aggregation conflicts")); - external.put("level", "Row"); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @Test - void explicitlyMarkedNativeAggregateDimensionKeepsItsSupportedLevel() { - Map target = target(); - field(target).put("dataType", "Number"); - Map external = calculation("external", "SUM([orders].[amount]) > 0"); - external.put("dataType", "Boolean"); - target.put("semanticCalculatedDimensions", List.of(external)); - assertTrue(error(source(), target).contains("expression aggregation conflicts")); - external.put("level", "AggregateFunction"); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @Test - void nativeOnlyCalculationDependenciesUseFinalTypesInTopologicalOrder() { - Map target = target(); - field(target).put("dataType", "Number"); - target.put("semanticCalculatedDimensions", List.of(calculation("row", "[orders].[amount] + 1"))); - target.put("semanticCalculatedMeasurements", List.of(calculation(target), calculation("external", "SUM([row])"))); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @Test - void nullOnlyNativeFormulaRequiresADeclaredTypeWithoutCrashing() { - Map target = target(); - Map external = calculation("external", "NULL"); - target.put("semanticCalculatedDimensions", List.of(external)); - assertTrue(error(source(), target).contains("dataType")); - external.put("dataType", "Text"); - assertDoesNotThrow(() -> validator.validate(source(), target)); - } - - @Test - void nativeArraysCannotDisappearBehindAlreadyPresentGeneratedArrays() { - Map source = source(); - source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"semanticCalculatedMeasurements\":[{\"apiName\":\"external\",\"expression\":\"1\",\"syntax\":\"Tua\"}]}"))); - assertTrue(error(source, target()).contains("Native extension semanticCalculatedMeasurements entity 'external' was not exported")); - Map target = target(); - target.put("semanticCalculatedMeasurements", List.of(calculation(target), calculation("external", "1"))); - assertDoesNotThrow(() -> validator.validate(source, target)); - // Matching core identities remain authoritative even when their native snapshot is stale. - source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"semanticCalculatedMeasurements\":[{\"apiName\":\"total\",\"expression\":\"STALE()\"}]}"))); - assertDoesNotThrow(() -> validator.validate(source, target())); - } - - @Test - void publicConverterRejectsUncompiledFunctionRestoredAtModelLevel() throws Exception { - String input = conversionInput("semanticCalculatedDimensions", - Map.of("apiName", "native_extra", "expression", "BOGUS(1)", "syntax", "Tua")); - ConversionException error = assertThrows(ConversionException.class, - () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); - assertTrue(error.getMessage().contains("native_extra"), error.getMessage()); - assertTrue(error.getMessage().contains("BOGUS"), error.getMessage()); - } - - @Test - void publicConverterRejectsNativeArrayLossAlongsideCoreMetrics() throws Exception { - String input = conversionInput("semanticCalculatedMeasurements", - Map.of("apiName", "native_extra", "expression", "1", "syntax", "Tua")); - ConversionException error = assertThrows(ConversionException.class, - () -> ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE).convert(input)); - assertTrue(error.getMessage().contains("native_extra"), error.getMessage()); - assertTrue(error.getMessage().contains("was not exported"), error.getMessage()); - } - - private static String conversionInput(String nativeArray, Map nativeCalculation) throws Exception { - com.fasterxml.jackson.databind.ObjectMapper json = new com.fasterxml.jackson.databind.ObjectMapper(); - Map source = source(); - sourceObject(source).put("source", "orders__dll"); - sourceObject(source).put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - "{\"dataObjectType\":\"Dlo\"}"))); - sourceObject(source).put("fields", List.of(Map.of("name", "amount", "datatype", "Decimal", "expression", - Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "amount__c")))))); - source.put("metrics", List.of(Map.of("name", "total", "datatype", "Decimal", "expression", - Map.of("dialects", List.of(Map.of("dialect", "SNOWFLAKE", "expression", "SUM(orders.amount)")))))); - source.put("custom_extensions", List.of(Map.of("vendor_name", "SALESFORCE", "data", - json.writeValueAsString(Map.of("dataspace", "default", nativeArray, List.of(nativeCalculation)))))); - return json.writeValueAsString(Map.of("version", "0.2.0.dev0", "semantic_model", List.of(source))); - } - - private String error(Map source, Map target) { - return assertThrows(ConversionException.class, () -> validator.validate(source, target)).getMessage(); - } - - private static Map source() { - Map source = new LinkedHashMap<>(); - source.put("name", "sales"); - source.put("datasets", new ArrayList<>(List.of(new LinkedHashMap<>(Map.of("name", "orders", "fields", List.of(Map.of("name", "amount"))))))); - source.put("metrics", List.of(Map.of("name", "total"))); - return source; - } - - private static Map target() { - Map target = new LinkedHashMap<>(); - target.put("apiName", "sales"); - target.put("semanticDataObjects", new ArrayList<>(List.of(new LinkedHashMap<>(Map.of("apiName", "orders", "dataObjectName", "orders__dll", - "semanticMeasurements", List.of(new LinkedHashMap<>(Map.of("apiName", "amount", "dataObjectFieldName", "amount__c")))))))); - target.put("semanticCalculatedMeasurements", List.of(calculation("total", "SUM([orders].[amount])"))); - return target; - } - - private static Map calculation(String name, String expression) { - return new LinkedHashMap<>(Map.of("apiName", name, "expression", expression, "syntax", "Tua")); - } - - @SuppressWarnings("unchecked") - private static List> datasets(Map source) { return (List>) source.get("datasets"); } - @SuppressWarnings("unchecked") - private static List> objects(Map target) { return (List>) target.get("semanticDataObjects"); } - private static Map sourceObject(Map source) { return datasets(source).get(0); } - private static Map object(Map target) { return objects(target).get(0); } - @SuppressWarnings("unchecked") - private static Map field(Map target) { return ((List>) object(target).get("semanticMeasurements")).get(0); } - @SuppressWarnings("unchecked") - private static Map calculation(Map target) { return ((List>) target.get("semanticCalculatedMeasurements")).get(0); } -} diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java deleted file mode 100644 index 3166d641..00000000 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/SourceDocumentValidationTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ossie.converter; - -import static org.junit.jupiter.api.Assertions.*; - -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.ossie.exception.InvalidInputException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -class SourceDocumentValidationTest { - @TempDir Path directory; - - @Test - void rejectsASecondOsiYamlDocumentRatherThanOmittingItsModel() throws Exception { - String valid = fixture("ossieToSalesforce.yaml"); - String second = valid.replace("Customer_Orders_Model", "Other_Model"); - assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, valid + "\n---\n" + second, "Trailing token"); - } - - @Test - void rejectsASecondNativeJsonObjectRatherThanOmittingIt() throws Exception { - String valid = fixture("salesforceToOssie.json"); - assertInvalid(ConversionDirection.SALESFORCE_TO_OSSIE, valid + "\n" + valid, "Trailing token"); - } - - @Test - void rejectsDuplicateYamlModelPropertiesBeforeTheyOverwriteTheFirstValue() throws Exception { - String input = fixture("ossieToSalesforce.yaml").replace(" - name: Customer_Orders_Model", - " - name: Customer_Orders_Model\n name: Silent_Replacement"); - assertTrue(input.contains("Silent_Replacement")); - assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, input, "Duplicate field 'name'"); - } - - @Test - void rejectsDuplicateNestedYamlTypesBeforeTheyChangeFieldMeaning() throws Exception { - String input = fixture("ossieToSalesforce.yaml").replaceFirst("(?m)^([ ]*)datatype: String$", - "$1datatype: String\n$1datatype: Integer"); - assertTrue(input.contains("datatype: Integer")); - assertInvalid(ConversionDirection.OSSIE_TO_SALESFORCE, input, "Duplicate field 'datatype'"); - } - - @Test - void rejectsDuplicateNativeJsonPropertiesBeforeTheyOverwriteTheFirstValue() throws Exception { - String valid = fixture("salesforceToOssie.json"); - String input = "{\"apiName\":\"Silently_Replaced\"," + valid.substring(valid.indexOf('{') + 1); - assertInvalid(ConversionDirection.SALESFORCE_TO_OSSIE, input, "Duplicate field 'apiName'"); - } - - @Test - void fileApiWritesNothingForTrailingSourceDocumentsAndAcceptsSingleDocumentComments() throws Exception { - String valid = fixture("ossieToSalesforce.yaml"); - Converter converter = ConverterFactory.getConverter(ConversionDirection.OSSIE_TO_SALESFORCE); - assertEquals(1, converter.convert(valid + "\n# trailing comments are part of the same document\n").size()); - Path input = directory.resolve("input.yaml"); - Files.writeString(input, valid + "\n---\nnull\n"); - Path existing = directory.resolve("Customer_Orders_Model.json"); - Files.writeString(existing, "preserve existing output"); - assertThrows(InvalidInputException.class, () -> converter.convert(input, directory)); - assertEquals("preserve existing output", Files.readString(existing)); - try (var files = Files.list(directory)) { assertEquals(2, files.count()); } - } - - private static void assertInvalid(ConversionDirection direction, String content, String message) { - InvalidInputException error = assertThrows(InvalidInputException.class, - () -> ConverterFactory.getConverter(direction).convert(content)); - assertTrue(error.getMessage().contains(message), error.getMessage()); - } - - private static String fixture(String file) throws Exception { - return Files.readString(Path.of("src/test/resources/examples", file)); - } -} diff --git a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml index df53a92b..101588f0 100644 --- a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml +++ b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml @@ -357,6 +357,35 @@ semantic_model: - product_id to_columns: - product_id + - name: Customers_ByDomain + from: Customers + to: Orders + from_columns: + - customer_email_domain + to_columns: + - order_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field" + } + - name: Orders_ByYear + from: Orders + to: Products + from_columns: + - order_year + to_columns: + - product_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field", + "cardinality" : "ManyToMany", + "joinType" : "Auto", + "isEnabled" : true + } metrics: - description: Sum of all order amounts name: total_revenue From 31aca84f108b3e1409faf434009595a63283a832 Mon Sep 17 00:00:00 2001 From: Saurabh Deshpande <43935865+saurabhdeshp@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:08:01 -0700 Subject: [PATCH 6/6] fix(salesforce): preserve redundant metric syntax and repair semantic tests Normalize bounded unary chains and redundant grouping before JSqlParser, while preserving operand validation, quotes, function arguments and limits. Reject malformed null predicates and tuple arguments explicitly. Fix ELSEIF evaluation and signed-zero equality, distinct counting and zero-divisor checks in the independent semantic test helper. Add focused compatibility, semantic and rejection regressions. Validation: 280 tests passed with no skips; Apache RAT passed. Packaged CLI checks cover successful conversion and failure without partial output. --- .../converter/SqlMetricExpressionParser.java | 72 +++++++++- .../MetricExpressionSemanticsTest.java | 124 ++++++++++++++++-- .../MetricExpressionTranslatorTest.java | 67 ++++++++++ 3 files changed, 253 insertions(+), 10 deletions(-) diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java index ef9e7583..3464ae33 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SqlMetricExpressionParser.java @@ -30,6 +30,8 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.AllColumns; +import org.apache.ossie.converter.TuaMetricExpressionParser.Kind; +import org.apache.ossie.converter.TuaMetricExpressionParser.Token; /** Adapts a completely consumed JSqlParser expression into the explicitly supported compiler AST. */ final class SqlMetricExpressionParser { @@ -38,7 +40,7 @@ final class SqlMetricExpressionParser { private SqlMetricExpressionParser(String dialect) { this.dialect = dialect; } static Node parse(String text, String dialect) { - TuaMetricExpressionParser.tokenize(text, dialect); + text = normalize(text, dialect); try { Expression expression = CCJSqlParserUtil.parseCondExpression(text, false, parser -> parser.withSquareBracketQuotation(dialect.equals("ANSI_SQL"))); @@ -52,6 +54,71 @@ static Node parse(String text, String dialect) { } } + /** Simplifies redundant syntax within the original input bounds, without reparsing SQL. */ + private static String normalize(String text, String dialect) { + List tokens = TuaMetricExpressionParser.tokenize(text, dialect); + int[] closing = new int[tokens.size()]; + int[] openings = new int[128]; + int nesting = 0; + for (int i = 0; i < tokens.size() - 1; i++) { + if (symbol(tokens.get(i), "(")) openings[nesting++] = i; + if (symbol(tokens.get(i), ")")) { + if (nesting == 0) throw new IllegalArgumentException("unexpected closing parenthesis"); + closing[openings[--nesting]] = i; + } + } + boolean[] redundant = new boolean[tokens.size()]; + for (int i = 1; i < tokens.size() - 2; i++) { + // Only remove a group inside another opening parenthesis. Keep the + // function argument list and innermost group, including tuple/modifier syntax. + if (symbol(tokens.get(i - 1), "(") && symbol(tokens.get(i), "(") + && symbol(tokens.get(i + 1), "(") && closing[i] == closing[i + 1] + 1) { + redundant[i] = redundant[closing[i]] = true; + } + } + StringBuilder result = new StringBuilder(text.length()); + for (int i = 0; i < tokens.size() - 1;) { + if (redundant[i]) { i++; continue; } + Token token = tokens.get(i); + boolean sign = symbol(token, "+") || symbol(token, "-"); + boolean not = token.kind() == Kind.WORD && token.text().equalsIgnoreCase("NOT"); + // Preserve binary +/- and predicate modifiers such as IS NOT. Only + // prefix operators can be simplified; malformed modifiers must still fail. + if ((not || sign) && (i == 0 || startsOperandAfter(tokens.get(i - 1)))) { + int end = i; + int negatives = 0; + while (end < tokens.size() - 1) { + Token next = tokens.get(end); + if (not ? next.kind() != Kind.WORD || !next.text().equalsIgnoreCase("NOT") + : !symbol(next, "+") && !symbol(next, "-")) break; + if (symbol(next, "-")) negatives++; + if (++end - i > 128) throw new IllegalArgumentException("too many unary operators"); + } + // Retain a unary operation even when parity is even: dropping all + // operators would bypass numeric/Boolean checks and COUNT(field) rules. + result.append(not ? (end - i) % 2 == 0 ? "NOT NOT " : "NOT " + : negatives % 2 == 0 ? "+ " : "- "); + i = end; + } else { + // Raw slices preserve escaped strings and quoted identifier spelling. + result.append(text, token.offset(), tokens.get(i + 1).offset()).append(' '); + i++; + } + } + return result.toString(); + } + + private static boolean symbol(Token token, String value) { + return token.kind() == Kind.SYMBOL && token.text().equals(value); + } + + private static boolean startsOperandAfter(Token token) { + return token.kind() == Kind.SYMBOL + && Set.of("(", ",", "+", "-", "*", "/", "=", "!=", "<>", "<", "<=", ">", ">=").contains(token.text()) + || token.kind() == Kind.WORD + && Set.of("WHEN", "THEN", "ELSE", "AND", "OR", "NOT", "DISTINCT").contains(token.text().toUpperCase(Locale.ROOT)); + } + private Node adapt(Expression expression) { if (++depth > 128) throw new IllegalArgumentException("expression nesting exceeds 128 levels"); try { return adaptNode(expression); } @@ -133,6 +200,9 @@ private Node function(Function function) { if (function.isAllColumns()) { throw new IllegalArgumentException("explicit ALL function modifier is outside the supported SQL subset"); } + if (function.getParameters() instanceof ParenthesedExpressionList grouped && grouped.size() != 1) { + throw new IllegalArgumentException("tuple-valued function arguments are unsupported"); + } List arguments = new ArrayList<>(); if (function.getParameters() != null) { for (Expression argument : function.getParameters()) arguments.add(adapt(argument)); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java index e6a09ba0..4e3fc8e4 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionSemanticsTest.java @@ -169,6 +169,94 @@ void textPredicatesPreserveDuplicatesNullsAndEscapedApostrophes(String dialect) assertValue(dialect, "COUNT(DISTINCT orders.status)", List.of(row(null, null, null, null)), 0.0); } + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void conditionalBranchesPreserveOrderAndImplicitNull(String dialect) { + List> orders = List.of( + row(10.0, 0.0, true), row(0.0, 0.0, false), + row(-4.0, 0.0, null), row(null, null, null)); + // Positive amounts match both of the first two conditions; the first must win. + assertValue(dialect, + "SUM(CASE WHEN orders.amount > 0 THEN 1 WHEN orders.amount >= 0 THEN 2 " + + "WHEN orders.amount < 0 THEN 3 ELSE 4 END)", + orders, 10.0); + assertValue(dialect, + "SUM(CASE WHEN orders.amount > 0 THEN 1 WHEN orders.amount >= 0 THEN 2 END)", + orders, 3.0); + assertValue(dialect, + "SUM(CASE WHEN orders.amount > 0 THEN 1 WHEN orders.amount = 0 THEN 2 END)", + List.of(row(-4.0, null, null), row(null, null, null)), null); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void nestedConditionalsSkipUnusedConditionsAndResults(String dialect) { + assertValue(dialect, + "SUM(CASE WHEN orders.amount > 0 THEN " + + "CASE WHEN orders.flag THEN 1 WHEN 1 / orders.cost > 0 THEN 99 ELSE 99 END " + + "WHEN orders.amount = 0 THEN 2 WHEN orders.amount < 0 THEN " + + "CASE WHEN orders.flag IS NULL THEN 3 ELSE 1 / orders.cost END ELSE 4 END)", + List.of(row(10.0, 0.0, true), row(0.0, 0.0, false), + row(-4.0, 0.0, null), row(null, 0.0, null)), 10.0); + assertValue(dialect, + "SUM(CASE WHEN orders.amount >= 0 THEN 1 " + + "WHEN 1 / orders.cost > 0 THEN 2 ELSE 1 / orders.cost END)", + List.of(row(10.0, 0.0, true), row(0.0, 0.0, false)), 2.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void numericComparisonsTreatSignedZerosAsEqual(String dialect) { + List> zeros = List.of(row(-0.0, null, null), row(0.0, null, null)); + assertValue(dialect, + "SUM(CASE WHEN orders.amount = 0 THEN 1 ELSE 0 END)", zeros, 2.0); + for (String operator : List.of("!=", "<>")) { + assertValue(dialect, + "SUM(CASE WHEN orders.amount " + operator + " 0 THEN 1 ELSE 0 END)", zeros, 0.0); + } + assertValue(dialect, + "SUM(CASE WHEN orders.amount <= 0 AND orders.amount >= 0 THEN 1 ELSE 0 END)", + zeros, 2.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void distinctCountTreatsSignedZerosAsOneValue(String dialect) { + List> orders = List.of( + row(-0.0, null, null), row(0.0, null, null), + row(1.0, null, null), row(null, null, null)); + assertValue(dialect, "COUNT(DISTINCT orders.amount)", orders, 2.0); + assertValue(dialect, "COUNT(orders.amount)", orders, 3.0); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void nullifGuardsEitherSignOfZero(String dialect) { + assertValue(dialect, "SUM(1 / NULLIF(orders.cost, 0))", + List.of(row(null, -0.0, null), row(null, 0.0, null)), null); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "-0"}) + void evaluatorRejectsEitherSignOfUnguardedZero(String zero) { + AssertionError error = assertThrows(AssertionError.class, + () -> new TuaSubsetEvaluator("1 / " + zero).evaluate(List.of())); + assertTrue(error.getMessage().contains("unguarded zero divisor")); + } + + @ParameterizedTest + @ValueSource(strings = {"SNOWFLAKE", "ANSI_SQL"}) + void repeatedUnaryOperatorsPreserveValuesAndUnknownPredicates(String dialect) { + List> orders = List.of( + row(3.0, null, true), row(7.0, null, false), + row(11.0, null, null), row(null, null, null)); + assertValue(dialect, "SUM(- -orders.amount)", orders, 21.0); + assertValue(dialect, + "SUM(CASE WHEN NOT NOT NOT orders.flag THEN orders.amount ELSE 0 END)", orders, 7.0); + assertValue(dialect, + "SUM(CASE WHEN NOT NOT NOT NOT orders.flag THEN orders.amount ELSE 0 END)", orders, 3.0); + } + private static void assertValue( String dialect, String sql, List> rows, Double expected) { Map metric = Map.of( @@ -263,13 +351,24 @@ private Calculation prefix() { return result; } if (token.equalsIgnoreCase("IF")) { - Calculation condition = expression(0); - expect("THEN"); - Calculation yes = expression(0); + List conditions = new ArrayList<>(); + List results = new ArrayList<>(); + do { + conditions.add(expression(0)); + expect("THEN"); + results.add(expression(0)); + } while (take("ELSEIF")); expect("ELSE"); - Calculation no = expression(0); + Calculation otherwise = expression(0); expect("END"); - return (rows, row) -> (Boolean.TRUE.equals(condition.value(rows, row)) ? yes : no).value(rows, row); + return (rows, row) -> { + for (int i = 0; i < conditions.size(); i++) { + if (Boolean.TRUE.equals(conditions.get(i).value(rows, row))) { + return results.get(i).value(rows, row); + } + } + return otherwise.value(rows, row); + }; } if (token.equalsIgnoreCase("NOT") || token.equals("-")) { Calculation child = expression(token.equals("-") ? 7 : 3); @@ -322,7 +421,9 @@ private static Calculation function(String name, List arguments) { return (double) values.size(); } if (name.equals("COUNTD")) { - return (double) values.stream().distinct().count(); + return (double) values.stream() + .map(value -> value instanceof Number && number(value) == 0.0 ? 0.0 : value) + .distinct().count(); } if (values.isEmpty()) { return null; @@ -384,11 +485,11 @@ private static Object binary(String operator, Object left, Object right) { case "-" -> number(left) - number(right); case "*" -> number(left) * number(right); case "/" -> { - assertNotEquals(0.0, number(right), "Generated expression evaluated an unguarded zero divisor"); + assertTrue(number(right) != 0.0, "Generated expression evaluated an unguarded zero divisor"); yield number(left) / number(right); } - case "=" -> left.equals(right); - case "!=", "<>" -> !left.equals(right); + case "=" -> equal(left, right); + case "!=", "<>" -> !equal(left, right); case "<" -> number(left) < number(right); case "<=" -> number(left) <= number(right); case ">" -> number(left) > number(right); @@ -397,6 +498,11 @@ private static Object binary(String operator, Object left, Object right) { }; } + private static boolean equal(Object left, Object right) { + return left instanceof Number && right instanceof Number + ? number(left) == number(right) : left.equals(right); + } + private static double number(Object value) { return ((Number) value).doubleValue(); } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java index 8ca36803..0692e89e 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/MetricExpressionTranslatorTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.*; +import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -84,6 +85,21 @@ static Stream supported() { Arguments.of("1 + 2 * 3 - 4 / 2", "((1 + (2 * 3)) - (4 / 2))"), Arguments.of(".25 + 1e2", "(0.25 + 100)"), Arguments.of("SUM(orders.quantity) + -2", "(SUM([orders].[quantity]) + (-2))"), + Arguments.of("SUM(- -orders.amount)", "SUM([orders].[amount])"), + Arguments.of("SUM(+ - - +orders.amount)", "SUM([orders].[amount])"), + Arguments.of("SUM(- + - -orders.amount)", "SUM((-[orders].[amount]))"), + Arguments.of("SUM(orders.amount - -orders.discount)", + "SUM(([orders].[amount] - (-[orders].[discount])))"), + Arguments.of("SUM(orders.amount) / - -SUM(orders.quantity)", + "(SUM([orders].[amount]) / SUM([orders].[quantity]))"), + Arguments.of("SUM((((orders.amount + orders.discount))) * 2)", + "SUM((([orders].[amount] + [orders].[discount]) * 2))"), + Arguments.of("SUM(CASE WHEN NOT NOT NOT orders.active THEN orders.amount ELSE 0 END)", + "SUM((IF (NOT [orders].[active]) THEN [orders].[amount] ELSE 0 END))"), + Arguments.of("SUM(CASE WHEN NOT NOT NOT NOT orders.active THEN orders.amount ELSE 0 END)", + "SUM((IF (NOT (NOT [orders].[active])) THEN [orders].[amount] ELSE 0 END))"), + Arguments.of("SUM(CASE WHEN orders.status = '- - NOT NOT NOT ((x))' THEN - -orders.amount ELSE 0 END)", + "SUM((IF ([orders].[status] = '- - NOT NOT NOT ((x))') THEN [orders].[amount] ELSE 0 END))"), Arguments.of("CASE WHEN MAX(orders.status) = 'z' THEN 1 ELSE 0 END", "(IF (MAX([orders].[status]) = 'z') THEN 1 ELSE 0 END)") ); @@ -252,6 +268,57 @@ void limitsNestingAndExpansionWithoutStackOverflow() { assertThrows(ConversionException.class, () -> translate("SNOWFLAKE", expanded)); } + @Test + void redundantParenthesesStayWithinTheBoundedFastParsingPath() { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { + for (int depth : List.of(20, 60, 126)) { + String grouped = "(".repeat(depth) + "orders.amount" + ")".repeat(depth); + assertEquals("SUM([orders].[amount])", translate(dialect, "SUM(" + grouped + ")")); + assertEquals("SUM([orders].[amount])", translate(dialect, + "(".repeat(depth) + "SUM(orders.amount)" + ")".repeat(depth))); + } + assertEquals("SUM([orders].[amount])", translate(dialect, "SUM(" + "- ".repeat(128) + "orders.amount)")); + assertEquals("SUM((IF (NOT (NOT [orders].[active])) THEN 1 ELSE 0 END))", + translate(dialect, "SUM(CASE WHEN " + "NOT ".repeat(128) + "orders.active THEN 1 ELSE 0 END)")); + } + }); + } + + @Test + void normalizationRetainsOriginalLimitsAndOperandTypes() { + for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { + for (String expression : List.of("SUM(" + "- ".repeat(129) + "orders.amount)", + "SUM(CASE WHEN " + "NOT ".repeat(129) + "orders.active THEN 1 ELSE 0 END)", + "SUM(" + "(".repeat(128) + "orders.amount" + ")".repeat(128) + ")", + "SUM(- -orders.status)", "COUNT(- -orders.amount)", + "SUM(CASE WHEN NOT NOT NOT NOT orders.amount THEN 1 ELSE 0 END)", + "SUM(CASE WHEN orders.amount IS NOT NOT NOT NULL THEN 1 ELSE 0 END)", + "SUM(CASE WHEN orders.amount IS NOT NOT NOT NOT NULL THEN 1 ELSE 0 END)", + "COUNT(((DISTINCT orders.amount)))", + "SUM(orders.amount)) + (1", "SUM(--orders.amount)")) { + assertTrue(assertThrows(ConversionException.class, () -> translate(dialect, expression), expression) + .getMessage().contains("Metric 'net_value':")); + } + } + } + + @Test + void redundantGroupingCannotTurnTuplesIntoFunctionArguments() { + for (String dialect : List.of("SNOWFLAKE", "ANSI_SQL")) { + for (String function : List.of("COALESCE", "ROUND")) { + for (int depth : List.of(1, 20)) { + String tuple = "(".repeat(depth) + "SUM(orders.amount), 2" + ")".repeat(depth); + assertTrue(assertThrows(ConversionException.class, + () -> translate(dialect, function + "(" + tuple + ")")) + .getMessage().contains("tuple-valued function arguments")); + } + } + assertEquals("IFNULL(SUM([orders].[amount]), 2)", + translate(dialect, "COALESCE((SUM(orders.amount)), 2)")); + } + } + @Test void temporalAggregatesCanBeUsedInNumericPredicates() { assertEquals("(IF (MIN([orders].[ordered]) = MAX([orders].[ordered])) THEN 1 ELSE 0 END)",