From 7279d3911613e5f323f3611162a8d525f184cb26 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Mon, 5 Oct 2026 18:42:35 +0200 Subject: [PATCH] fix paged SQL feature queries with an id list returning all matches on every page Value queries skipped the key-range window whenever the filter contained a feature id list. That is only correct for single-feature queries, which run without a meta query; a paged query with several ids returned every match in every chunk and on every page. The window is now omitted only when the meta query is skipped. --- .../sql/app/FeatureQueryEncoderSql.java | 11 ++++---- .../sql/app/SqlQueryTemplatesDeriver.java | 26 ++----------------- .../app/SqlQueryTemplatesDeriverSpec.groovy | 2 ++ .../sql-queries/object_array_ids.yml | 12 +++++++++ .../sql-queries/object_array_ids_paging.yml | 12 +++++++++ 5 files changed, 34 insertions(+), 29 deletions(-) create mode 100644 xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids.yml create mode 100644 xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids_paging.yml diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java index 9fd955923..56f1f2843 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FeatureQueryEncoderSql.java @@ -229,11 +229,12 @@ private SqlQuerySet createQuerySet( queryTemplates.getValueQueryTemplates().stream() .map( valueQueryTemplate -> - // single-shot reads matching rows in one pass: no offset and no key-range - // window (which would otherwise constrain the result set to the meta - // query's minKey/maxKey); an optional per-sub-query maximum caps each - // sub-query (0 = no limit) - unpaged + // single-shot reads, and reads without a meta query (single feature), + // read matching rows in one pass: no offset and no key-range window + // (which would otherwise constrain the result set to the meta query's + // minKey/maxKey); an optional per-sub-query maximum caps each sub-query + // (0 = no limit) + unpaged || skipMetaQuery ? valueQueryTemplate.generateValueQuery( query instanceof MultiFeatureQuery ? ((MultiFeatureQuery) query).getMaxFeaturesPerSubQuery() diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriver.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriver.java index 9e81376a6..5a087c650 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriver.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriver.java @@ -10,7 +10,6 @@ import com.google.common.collect.ImmutableList; import de.ii.xtraplatform.cql.domain.And; import de.ii.xtraplatform.cql.domain.Cql2Expression; -import de.ii.xtraplatform.cql.domain.In; import de.ii.xtraplatform.features.domain.SortKey; import de.ii.xtraplatform.features.domain.SortKey.Direction; import de.ii.xtraplatform.features.domain.Tuple; @@ -166,17 +165,14 @@ ValueQueryTemplate createValueQueryTemplate(SqlQuerySchema schema, SqlQueryMappi forceSimpleFeatures, minMaxKeys, virtualTables) -> { - boolean isIdFilter = filter.filter(SqlQueryTemplatesDeriver::containsIdFilter).isPresent(); List aliases = AliasGenerator.getAliases(schema); SqlQueryTable main = schema.getRelations().isEmpty() ? schema : schema.getRelations().get(0); Optional sqlFilter = getFilter(main, mapping, filter); Optional whereClause = - isIdFilter - ? sqlFilter - : toWhereClause( - aliases.get(0), main.getSortKey(), additionalSortKeys, minMaxKeys, sqlFilter); + toWhereClause( + aliases.get(0), main.getSortKey(), additionalSortKeys, minMaxKeys, sqlFilter); boolean useRangePaging = additionalSortKeys.isEmpty() && minMaxKeys.isPresent(); Optional pagingClause = useRangePaging || (limit == 0 && offset == 0) @@ -282,24 +278,6 @@ private String getTableQuery( columns, mainTable, join.isEmpty() ? "" : " ", join, where, orderBy, paging); } - /** - * Recognises an id-bounded filter even when it is buried inside conjunctions, e.g. {@code - * In(_ID_, [...])} on its own or {@code And(In(_ID_, [...]), )} — both treat the row-set - * as constrained by the id list and let the SQL generator skip the surrogate-key range guard. - */ - private static boolean containsIdFilter(Cql2Expression expr) { - if (expr instanceof In && ((In) expr).isIdFilter()) { - return true; - } - if (expr instanceof And) { - return ((And) expr) - .getArgs().stream() - .anyMatch( - arg -> arg instanceof Cql2Expression && containsIdFilter((Cql2Expression) arg)); - } - return false; - } - private Optional toWhereClause( String alias, String keyField, diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriverSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriverSpec.groovy index cebdf7779..0b698eafd 100644 --- a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriverSpec.groovy +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/SqlQueryTemplatesDeriverSpec.groovy @@ -104,6 +104,8 @@ class SqlQueryTemplatesDeriverSpec extends Specification { "self joins with nested duplicate join" | td | 0 | 0 | [] | null | "self_joins_with_nested_duplicate_join" | "self_joins_with_nested_duplicate_join" || "self_joins_with_nested_duplicate_join" "object without sourcePath" | td | 0 | 0 | [] | null | "object_without_sourcePath" | "object_without_sourcePath" || "object_without_sourcePath" "paging" | td | 10 | 10 | [] | null | "object_array" | "object_array" || "object_array_paging" + "ids" | td | 0 | 0 | [] | In.of(ScalarLiteral.of("1"), ScalarLiteral.of("2")) | "object_array" | "object_array" || "object_array_ids" + "ids + paging" | td | 10 | 10 | [] | In.of(ScalarLiteral.of("1"), ScalarLiteral.of("2")) | "object_array" | "object_array" || "object_array_ids_paging" "sortBy" | td | 0 | 0 | [SortKey.of("created")] | null | "object_array" | "object_array" || "object_array_sortby" "sortBy + filter" | td | 0 | 0 | [SortKey.of("created")] | Eq.of(Property.of("task.title"), ScalarLiteral.of("foo")) | "object_array" | "object_array" || "object_array_sortby_filter" "sortBy + paging" | td | 10 | 10 | [SortKey.of("created")] | null | "object_array" | "object_array" || "object_array_sortby_paging" diff --git a/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids.yml b/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids.yml new file mode 100644 index 000000000..3db35220a --- /dev/null +++ b/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids.yml @@ -0,0 +1,12 @@ +- >- + SELECT A.id AS SKEY, A.id + FROM explorationsite A + WHERE (A.id IN ('1','2')) + ORDER BY 1 +- >- + SELECT A.id AS SKEY, C.id AS SKEY_1, C.id, C.projectname, C.id + FROM explorationsite A + JOIN explorationsite_task B ON (A.id=B.explorationsite_fk) + JOIN task C ON (B.task_fk=C.id) + WHERE (A.id IN ('1','2')) + ORDER BY 1,2 diff --git a/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids_paging.yml b/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids_paging.yml new file mode 100644 index 000000000..11847eb91 --- /dev/null +++ b/xtraplatform-features-sql/src/test/resources/sql-queries/object_array_ids_paging.yml @@ -0,0 +1,12 @@ +- >- + SELECT A.id AS SKEY, A.id + FROM explorationsite A + WHERE (A.id >= 10 AND A.id <= 19) AND (A.id IN ('1','2')) + ORDER BY 1 +- >- + SELECT A.id AS SKEY, C.id AS SKEY_1, C.id, C.projectname, C.id + FROM explorationsite A + JOIN explorationsite_task B ON (A.id=B.explorationsite_fk) + JOIN task C ON (B.task_fk=C.id) + WHERE (A.id >= 10 AND A.id <= 19) AND (A.id IN ('1','2')) + ORDER BY 1,2