From 3e94b6f7a686ceabff2f6f48381aaee8b8578591 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:08:23 -0700 Subject: [PATCH 01/14] refactor(aggregates): extract lateral aggregate loading Move the loaded-aggregate implementation into `AshSql.Aggregate.Lateral` and leave `AshSql.Aggregate` as the compatibility facade. Implementation notes: - preserve the existing public function signatures; - delegate directly to the extracted lateral module; - retain the upstream aggregate read-action correction; - avoid the temporary context wrapper from the earlier history. This is a structural extraction with no intended behavior change. --- lib/aggregate.ex | 2949 +------------------------------------ lib/aggregate/lateral.ex | 2954 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 2995 insertions(+), 2908 deletions(-) create mode 100644 lib/aggregate/lateral.ex diff --git a/lib/aggregate.ex b/lib/aggregate.ex index b532436..c59b5e2 100644 --- a/lib/aggregate.ex +++ b/lib/aggregate.ex @@ -5,14 +5,6 @@ defmodule AshSql.Aggregate do @moduledoc false - require Ecto.Query - require Ash.Query - import Ecto.Query, only: [from: 2, subquery: 1] - - @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> - Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") - end) - def add_aggregates( query, aggregates, @@ -22,2934 +14,75 @@ defmodule AshSql.Aggregate do root_data \\ nil ) - def add_aggregates(query, [], _, _, _, _), do: {:ok, query} + def add_aggregates(query, [], _resource, _select?, _source_binding, _root_data), + do: {:ok, query} def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do - case resource_aggregates_to_aggregates(resource, query, aggregates) do - {:ok, aggregates} -> - root_data_path = - case root_data do - {_, path} -> - path - - _ -> - [] - end - - tenant = - case Enum.at(aggregates, 0) do - %{context: %{tenant: tenant}} -> - Ash.ToTenant.to_tenant(tenant, resource) - - _ -> - nil - end - - {query, aggregates} = - Enum.reduce( - aggregates, - {query, []}, - fn aggregate, {query, aggregates} -> - if is_atom(aggregate.name) do - existing_agg = query.__ash_bindings__.aggregate_defs[aggregate.name] - - if existing_agg && different_queries?(existing_agg.query, aggregate.query) do - {query, name} = use_aggregate_name(query, aggregate.name) - {query, [%{aggregate | name: name} | aggregates]} - else - {query, [aggregate | aggregates]} - end - else - {query, name} = use_aggregate_name(query, aggregate.name) - - {query, [%{aggregate | name: name} | aggregates]} - end - end - ) - - {already_computed_aggregates, remaining_aggregates} = - aggregates - |> Enum.uniq_by(& &1.name) - |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, [])) - - query = - if Enum.any?(already_computed_aggregates) && select? do - query.__ash_bindings__.bindings - |> Enum.filter(fn - {_binding, %{type: :aggregate}} -> true - _ -> false - end) - |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> - q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) - - Enum.reduce(aggs, q, fn agg, q -> - if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do - q = - update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> - [agg.name | select_aggs] - end) - - if agg.default_value do - from(row in q, - select_merge: %{ - ^agg.name => - coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) - } - ) - else - from(row in q, - select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} - ) - end - else - q - end - end) - end) - else - query - end - - query = - if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && - !query.__ash_bindings__[:lateral_join?] && - Enum.any?( - remaining_aggregates, - &(not optimizable_first_aggregate?(resource, &1, query)) - ) do - wrap_in_subquery_for_aggregates(query) - else - query - end - - query = - if root_data_path == [] do - query - |> Map.update!(:__ash_bindings__, fn bindings -> - bindings - |> Map.update!(:aggregate_defs, fn aggregate_defs -> - Map.merge(aggregate_defs, Map.new(aggregates, &{&1.name, &1})) - end) - end) - else - query - end - - result = - remaining_aggregates - |> Enum.group_by(fn aggregate -> - expanded_path = - aggregate.resource - |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) - |> Enum.map(& &1.name) - - {expanded_path, aggregate.resource, aggregate.join_filters || %{}, - aggregate.query.action.name} - end) - |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> - {can_group, cant_group} = - Enum.split_with(aggregates, &can_group?(resource, &1, query)) - - [{{path, resource, join_filters, read_action}, can_group}] ++ - Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) - end) - |> Enum.reject(fn - {_, []} -> - true - - _ -> - false - end) - |> Enum.reduce_while( - {:ok, query, []}, - fn {{path, resource, join_filters, read_action}, aggregates}, - {:ok, query, dynamics} -> - related = Ash.Resource.Info.related(resource, path) - read_action = Ash.Resource.Info.action(related, read_action) - - if read_action.modify_query do - raise """ - Data layer does not currently support aggregates over read actions that use `modify_query`. - - Resource: #{inspect(resource)} - Relationship Path: #{inspect(path)} - Action: #{read_action.name} - """ - end - - {first_relationship, relationship_path} = - case path do - [] -> - {nil, []} - - [first_relationship | rest] -> - case Ash.Resource.Info.relationship(resource, first_relationship) do - nil -> - raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" - - first_relationship -> - if rest == [] do - {override_read_action(first_relationship, read_action.name), rest} - else - {first_relationship, rest} - end - end - end - - hydrated_agg_refs = - aggregates - |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) - |> Ash.Filter.hydrate_refs(%{ - resource: Enum.at(aggregates, 0).query.resource, - parent_stack: - if(first_relationship, do: [first_relationship.source], else: [resource]) - }) - |> elem(1) - - parent_expr = - if first_relationship do - first_relationship.filter - |> Ash.Filter.hydrate_refs(%{ - resource: first_relationship.destination, - parent_stack: [first_relationship.source] - }) - |> elem(1) - |> then(&[&1 | hydrated_agg_refs]) - |> AshSql.Join.parent_expr() - end - - used_aggregates = - Ash.Filter.used_aggregates(parent_expr, []) - - {:ok, query} = - AshSql.Aggregate.add_aggregates( - query, - used_aggregates, - resource, - false, - query.__ash_bindings__.root_binding - ) - - {:ok, query} = - AshSql.Join.join_all_relationships( - query, - parent_expr, - [], - nil, - [], - nil, - true, - nil, - nil, - true - ) - - is_single? = match?([_], aggregates) - - cond do - is_single? && - optimizable_first_aggregate?( - resource, - Enum.at(aggregates, 0), - query - ) -> - case add_first_join_aggregate( - query, - resource, - hd(aggregates), - root_data, - first_relationship, - source_binding - ) do - {:ok, query, dynamic} -> - query = - if select? do - select_or_merge(query, hd(aggregates).name, dynamic) - else - query - end - - {:cont, {:ok, query, dynamics}} - - {:error, error} -> - {:halt, {:error, error}} - end - - is_single? && Enum.at(aggregates, 0).kind == :exists -> - [aggregate] = aggregates - - expr = - if is_nil(Map.get(aggregate.query, :filter)) do - true - else - Map.get(aggregate.query, :filter) - end - - {exists, acc} = - AshSql.Expr.dynamic_expr( - query, - %Ash.Query.Exists{ - path: root_data_path ++ aggregate.relationship_path, - related?: aggregate.related?, - resource: aggregate.query.resource, - expr: expr - }, - query.__ash_bindings__ - ) - - {:cont, - {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), - [{aggregate.load, aggregate.name, exists} | dynamics]}} - - true -> - tmp_query = - if first_relationship && first_relationship.type == :many_to_many do - put_in(query.__ash_bindings__[:lateral_join_bindings], [ - query.__ash_bindings__.current - ]) - |> AshSql.Bindings.explicitly_set_binding( - %{ - type: :left, - path: [first_relationship.join_relationship] - }, - query.__ash_bindings__.current - ) - else - query - end - - start_bindings_at = - if first_relationship && first_relationship.type == :many_to_many do - query.__ash_bindings__.current + 1 - else - query.__ash_bindings__.current - end - - case get_subquery( - resource, - aggregates, - is_single?, - first_relationship, - relationship_path, - tmp_query, - start_bindings_at, - query, - source_binding, - root_data_path, - tenant, - join_filters - ) do - {:error, error} -> - {:error, error} - - {:ok, subquery} -> - query = - join_subquery( - query, - subquery, - first_relationship, - relationship_path, - aggregates, - source_binding, - root_data_path - ) - - if select? do - new_dynamics = - Enum.map( - aggregates, - &{&1.load, &1.name, - select_dynamic( - resource, - query, - &1, - query.__ash_bindings__.current - 1 - )} - ) - - {:cont, {:ok, query, new_dynamics ++ dynamics}} - else - {:cont, {:ok, query, dynamics}} - end - end - end - end - ) - - case result do - {:ok, query, dynamics} -> - if select? do - {:ok, add_aggregate_selects(query, dynamics)} - else - {:ok, query} - end - - {:error, error} -> - {:error, error} - end - - {:error, error} -> - {:error, error} - end - end - - defp already_added?(aggregate, bindings, root_data_path) do - Enum.any?(bindings.bindings, fn - {_, %{type: :aggregate, aggregates: aggregates, path: ^root_data_path}} -> - aggregate.name in Enum.map(aggregates, & &1.name) - - _other -> - false - end) - end - - defp get_subquery( - _resource, - aggregates, - is_single?, - nil, - _relationship_path, - _tmp_query, - start_bindings_at, - query, - _source_binding, - root_data_path, - tenant, - _join_filters - ) do - first_aggregate = Enum.at(aggregates, 0) - aggregate_resource = first_aggregate.query.resource - - first_aggregate.query - |> Ash.Query.set_context(%{ - data_layer: %{ - table: nil, - parent_bindings: - Map.put( - query.__ash_bindings__, - :refs_at_path, - root_data_path - ), - start_bindings_at: start_bindings_at || 0 - } - }) - |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) - |> AshSql.Join.handle_attribute_multitenancy(tenant) - |> AshSql.Join.hydrate_refs(query.__ash_bindings__.context[:private][:actor]) - |> case do - %{valid?: true} = related_query -> - case Ash.Query.data_layer_query(related_query) do - {:ok, ecto_query} -> - {:ok, Ecto.Query.exclude(ecto_query, :select)} - - {:error, error} -> - {:error, error} - end - - %{errors: errors} -> - {:error, errors} - end - |> case do - {:ok, query} -> - maybe_filter_subquery( - query, - nil, - [], - aggregates, - is_single?, - query.__ash_bindings__.root_binding - ) - - {:error, error} -> - {:error, error} - end - |> case do - {:error, error} -> - {:error, error} - - {:ok, query} -> - if is_single? and has_filter?(Enum.at(aggregates, 0).query) do - AshSql.Filter.filter( - query, - Enum.at(aggregates, 0).query.filter, - aggregate_resource - ) - else - {:ok, query} - end - |> case do - {:error, error} -> - {:error, error} - - {:ok, filtered} -> - filtered = - AshSql.Join.set_join_prefix( - filtered, - %{query | prefix: tenant}, - aggregate_resource - ) - - {:ok, - select_all_aggregates( - aggregates, - filtered, - [], - query, - is_single?, - aggregate_resource, - nil - )} - end - end - end - - defp get_subquery( - resource, - aggregates, - is_single?, - first_relationship, - relationship_path, - tmp_query, - start_bindings_at, - query, - source_binding, - root_data_path, - tenant, - join_filters - ) do - limited? = limited_relationship?(first_relationship) - - limit_one_first? = - is_single? && relationship_path == [] && - limit_one_first_aggregate?(resource, Enum.at(aggregates, 0)) - - AshSql.Join.related_subquery( - first_relationship, - tmp_query, - start_bindings_at: start_bindings_at, - refs_at_path: root_data_path, - skip_distinct_for_first_rel?: true, - sort?: limited?, - on_subquery: fn subquery -> - base_binding = subquery.__ash_bindings__.root_binding - current_binding = subquery.__ash_bindings__.current - - subquery = - cond do - limited? -> - apply_limited_relationship_subquery( - subquery, - first_relationship, - source_binding, - base_binding - ) - - limit_one_first? -> - apply_limit_one_first_subquery( - subquery, - Enum.at(aggregates, 0), - first_relationship, - source_binding, - base_binding - ) - - true -> - subquery - |> Ecto.Query.exclude(:select) - |> Ecto.Query.select(%{}) - |> apply_relationship_subquery( - first_relationship, - query, - tenant, - source_binding, - current_binding, - base_binding - ) - end - - subquery = - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - first_relationship.destination - ) - - {:ok, subquery, _} = - apply_first_relationship_join_filters( - subquery, - query, - %AshSql.Expr.ExprInfo{}, - first_relationship, - join_filters - ) - - subquery = - set_in_group( - subquery, - query, - resource - ) - - {:ok, joined} = - join_all_relationships( - subquery, - aggregates, - relationship_path, - first_relationship, - is_single?, - join_filters - ) - - # When the aggregate's filter has already been pushed into the - # `LIMIT 1` inner subquery, it must not be applied again here. - {:ok, filtered} = - maybe_filter_subquery( - joined, - first_relationship, - relationship_path, - aggregates, - is_single? && !limit_one_first?, - subquery.__ash_bindings__.root_binding - ) - - select_all_aggregates( - aggregates, - filtered, - relationship_path, - query, - is_single?, - Ash.Resource.Info.related( - first_relationship.destination, - relationship_path - ), - first_relationship - ) - end - ) - end - - # Relationships that declare a `limit` (or `offset`) need the limit applied - # to the correlated rows *before* the aggregation's `GROUP BY`, otherwise the - # limit caps the number of groups (always 1 in a lateral join) instead of the - # number of rows per group. - defp limited_relationship?(relationship) do - (is_integer(Map.get(relationship, :limit)) or - (Map.get(relationship, :offset) || 0) > 0) and - is_nil(Map.get(relationship, :manual)) and - !Map.get(relationship, :no_attributes?) and - relationship.type != :many_to_many - end - - # Builds: - # - # SELECT ... FROM ( - # SELECT * FROM destination - # WHERE destination.destination_attribute = parent.source_attribute - # ORDER BY LIMIT - # ) AS - # GROUP BY destination_attribute - # - # The correlation, sort and limit all live in the inner subquery so that the - # limit bounds the rows per parent, and the aggregate functions fold the - # already-limited rows. - defp apply_limited_relationship_subquery(subquery, rel, source_binding, base_binding) do - field = rel.destination_attribute - - inner = - from(row in subquery, - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - as(^base_binding), - ^rel.destination_attribute - ) - ) - - inner = - case Map.get(rel, :limit) do - limit when is_integer(limit) -> Ecto.Query.limit(inner, ^limit) - _ -> inner - end - - from(row in subquery(inner), as: ^base_binding) - |> Map.put(:__ash_bindings__, subquery.__ash_bindings__) - |> Ecto.Query.select(%{}) - |> then(fn wrapped -> - from(row in wrapped, - group_by: field(row, ^field), - select_merge: %{^field => field(row, ^field)} - ) - end) - end - - @doc false - def limit_one_first_aggregate?( - resource, - %{ - kind: :first, - relationship_path: [relationship_name], - field: field, - join_filters: join_filters - } = aggregate - ) - when is_atom(field) and not is_nil(field) do - case Ash.Resource.Info.relationship(resource, relationship_name) do - nil -> - false - - relationship -> - relationship.type != :many_to_many && - is_nil(Map.get(relationship, :manual)) && - !Map.get(relationship, :no_attributes?) && - !limited_relationship?(relationship) && - join_filters in [nil, %{}, []] && - (has_sort?(aggregate.query) || relationship.sort not in [nil, []]) && - match?( - %Ash.Resource.Attribute{}, - Ash.Resource.Info.attribute(relationship.destination, field) - ) - end - end - - def limit_one_first_aggregate?(_resource, _aggregate), do: false - - defp apply_limit_one_first_subquery( - subquery, - aggregate, - rel, - source_binding, - base_binding - ) do - group_field = rel.destination_attribute - - inner = - from(row in subquery, - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - as(^base_binding), - ^rel.destination_attribute - ) - ) - - {:ok, inner} = - if has_filter?(aggregate.query) do - AshSql.Filter.filter(inner, aggregate.query.filter, rel.destination) - else - {:ok, inner} - end - - inner = - if aggregate.include_nil? do - inner - else - from(row in inner, - where: not is_nil(field(as(^base_binding), ^aggregate.field)) - ) - end - - sort = - if has_sort?(aggregate.query) do - aggregate.query.sort - else - List.wrap(rel.sort) - end - - {:ok, inner} = - AshSql.Sort.sort(inner, sort, rel.destination, [], base_binding, :direct) - - inner = Ecto.Query.limit(inner, 1) - - from(row in subquery(inner), as: ^base_binding) - |> Map.put(:__ash_bindings__, subquery.__ash_bindings__) - |> Ecto.Query.select(%{}) - |> then(fn wrapped -> - from(row in wrapped, - group_by: field(row, ^group_field), - select_merge: %{^group_field => field(row, ^group_field)} - ) - end) - end - - defp apply_relationship_subquery( - subquery, - %{manual: {module, opts}} = rel, - query, - tenant, - source_binding, - current_binding, - _base_binding - ) do - field = rel.destination_attribute - - from(row in subquery, - group_by: field(row, ^field), - select_merge: %{^field => field(row, ^field)} - ) - - subquery = - from(row in subquery, distinct: true) - - {:ok, subquery} = - apply( - module, - query.__ash_bindings__.sql_behaviour.manual_relationship_subquery_function(), - [ - opts, - source_binding, - current_binding - 1, - subquery - ] - ) - - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - rel.destination - ) - end - - defp apply_relationship_subquery( - subquery, - %{no_attributes?: true}, - _query, - _tenant, - _source_binding, - _current_binding, - _base_binding - ) do - subquery - end - - defp apply_relationship_subquery( - subquery, - %{type: :many_to_many} = rel, - query, - tenant, - source_binding, - _current_binding, - _base_binding - ) do - join_relationship_struct = - Ash.Resource.Info.relationship( - rel.source, - rel.join_relationship - ) - - {:ok, through} = - AshSql.Join.related_subquery( - join_relationship_struct, - query - ) - - field = rel.source_attribute_on_join_resource - - subquery = - from(sub in subquery, - join: through in ^through, - as: ^query.__ash_bindings__.current, - on: - field( - through, - ^rel.destination_attribute_on_join_resource - ) == - field(sub, ^rel.destination_attribute), - select_merge: map(through, ^[field]), - group_by: - field( - through, - ^rel.source_attribute_on_join_resource - ), - distinct: - field( - through, - ^rel.source_attribute_on_join_resource - ), - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - through, - ^rel.source_attribute_on_join_resource - ) - ) - - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - rel.destination - ) - end - - defp apply_relationship_subquery( - subquery, - rel, - _query, - _tenant, - source_binding, - _current_binding, - base_binding - ) do - field = rel.destination_attribute - - from(row in subquery, - group_by: field(row, ^field), - select_merge: %{^field => field(row, ^field)}, - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - as(^base_binding), - ^rel.destination_attribute - ) - ) - end - - defp set_in_group(%{__ash_bindings__: _} = query, _, _resource) do - Map.update!( + AshSql.Aggregate.Lateral.add_aggregates( query, - :__ash_bindings__, - &Map.put(&1, :in_group?, true) - ) - end - - defp set_in_group(%Ecto.SubQuery{} = subquery, query, resource) do - subquery = from(row in subquery, []) - - subquery - |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) - |> Map.update!( - :__ash_bindings__, - &Map.put(&1, :in_group?, true) + aggregates, + resource, + select?, + source_binding, + root_data ) end - defp set_in_group(other, query, resource) do - from(row in other, as: ^0) - |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) - |> Map.update!( - :__ash_bindings__, - &Map.put(&1, :in_group?, true) - ) - end - - defp different_queries?(nil, nil), do: false - defp different_queries?(nil, _), do: true - defp different_queries?(_, nil), do: true - - defp different_queries?(query1, query2) do - query1.filter != query2.filter || query1.sort != query2.sort - end - - @doc false def extract_shared_filters(aggregates) do - aggregates - |> Enum.reduce_while({nil, []}, fn - %{query: %{filter: filter}} = agg, {global_filters, aggs} when not is_nil(filter) -> - and_statements = - AshSql.Expr.split_statements(filter, :and) - - global_filters = - if global_filters do - Enum.filter(global_filters, &(&1 in and_statements)) - else - and_statements - end - - {:cont, {global_filters, [{agg, and_statements} | aggs]}} - - _, _ -> - {:halt, {:error, aggregates}} - end) - |> case do - {:error, aggregates} -> - {:error, aggregates} - - {[], _} -> - {:error, aggregates} - - {nil, _} -> - {:error, aggregates} - - {global_filters, aggregates} -> - global_filter = and_filters(Enum.uniq(global_filters)) - - aggregates = - Enum.map(aggregates, fn {agg, and_statements} -> - applicable_and_statements = - and_statements - |> Enum.reject(&(&1 in global_filters)) - |> and_filters() - - %{agg | query: %{agg.query | filter: applicable_and_statements}} - end) - - {{:ok, global_filter}, aggregates} - end - end - - defp and_filters(filters) do - Enum.reduce(filters, nil, fn expr, acc -> - if is_nil(acc) do - expr - else - Ash.Query.BooleanExpression.new(:and, expr, acc) - end - end) + AshSql.Aggregate.Lateral.extract_shared_filters(aggregates) end - defp apply_first_relationship_join_filters( - agg_root_query, - query, - acc, - first_relationship, - join_filters - ) do - case join_filters[[first_relationship.name]] do - nil -> - {:ok, agg_root_query, acc} - - filter -> - with {:ok, agg_root_query} <- - AshSql.Join.join_all_relationships(agg_root_query, filter) do - agg_root_query = - AshSql.Expr.set_parent_path( - agg_root_query, - query - ) - - {query, acc} = - AshSql.Join.maybe_apply_filter( - agg_root_query, - agg_root_query, - agg_root_query.__ash_bindings__, - filter - ) - - {:ok, query, acc} - end - end + def next_aggregate_name(index) do + AshSql.Aggregate.Lateral.next_aggregate_name(index) end - defp use_aggregate_name(query, aggregate_name) do - {%{ - query - | __ash_bindings__: %{ - query.__ash_bindings__ - | current_aggregate_name: - next_aggregate_name(query.__ash_bindings__.current_aggregate_name), - aggregate_names: - Map.put( - query.__ash_bindings__.aggregate_names, - aggregate_name, - query.__ash_bindings__.current_aggregate_name - ) - } - }, query.__ash_bindings__.current_aggregate_name} + def can_group?(resource, aggregate, query) do + AshSql.Aggregate.Lateral.can_group?(resource, aggregate, query) end - defp resource_aggregates_to_aggregates(resource, query, aggregates) do - private_context = query.__ash_bindings__.context[:private] - - Enum.reduce_while(aggregates, {:ok, []}, fn - %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> - aggregate = - Ash.Actions.Read.add_calc_context( - aggregate, - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - aggregate, {:ok, aggregates} -> - resource - |> resource_aggregate_to_aggregate(aggregate, - actor: private_context[:actor], - tenant: private_context[:tenant] - ) - |> case do - {:ok, aggregate} -> - aggregate = - aggregate - |> Map.put(:load, aggregate.name) - |> Ash.Actions.Read.add_calc_context( - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - {:error, error} -> - {:halt, {:error, error}} - end - end) + def optimizable_first_aggregate?(resource, aggregate, query) do + AshSql.Aggregate.Lateral.optimizable_first_aggregate?(resource, aggregate, query) end - @doc false - def resource_aggregate_to_aggregate(resource, aggregate, opts \\ []) do - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - read_action = - aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name - - with %{valid?: true} = aggregate_query <- - Ash.Query.for_read(related, read_action, %{}, - actor: opts[:actor], - tenant: opts[:tenant] - ), - %{valid?: true} = aggregate_query <- - Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do - Ash.Query.Aggregate.new( + def add_subquery_aggregate_select( + query, + relationship_path, + aggregate, resource, - aggregate.name, - aggregate.kind, - path: aggregate.relationship_path, - query: aggregate_query, - field: aggregate.field, - default: aggregate.default, - filterable?: aggregate.filterable?, - type: aggregate.type, - sortable?: aggregate.filterable?, - include_nil?: aggregate.include_nil?, - constraints: aggregate.constraints, - implementation: aggregate.implementation, - uniq?: aggregate.uniq?, - read_action: read_action, - authorize?: aggregate.authorize? - ) - else - %{errors: errors} -> - {:error, errors} - end - end - - defp add_first_join_aggregate( - query, - _resource, - %{related?: false} = aggregate, - root_data, - _, - source_binding - ) do - path = - case root_data do - {_resource, path} -> - path - - _ -> - [] - end - - subquery_result = - aggregate.query - |> Ash.Query.set_context(%{ - data_layer: %{ - table: nil, - parent_bindings: - Map.put( - query.__ash_bindings__, - :refs_at_path, - path - ), - start_bindings_at: (query.__ash_bindings__.current || 0) + 1 - } - }) - |> Ash.Query.limit(1) - |> Ash.Query.data_layer_query() - - case subquery_result do - {:ok, ecto_query} -> - ref = - %Ash.Query.Ref{ - attribute: aggregate.field, - resource: aggregate.query.resource - } - - {:ok, ecto_query} = AshSql.Join.join_all_relationships(ecto_query, ref) - - ecto_query = - case aggregate.field do - %Ash.Query.Aggregate{} = aggregate -> - {:ok, ecto_query} = - add_aggregates( - ecto_query, - [aggregate], - aggregate.query.resource, - true, - source_binding, - root_data - ) - - ecto_query - - %Ash.Resource.Aggregate{} = aggregate -> - {:ok, ecto_query} = - add_aggregates( - ecto_query, - [aggregate], - Ash.Resource.Info.related(aggregate.resource, aggregate.relationship_path), - true, - source_binding, - root_data - ) - - ecto_query - - %Ash.Resource.Calculation{ - name: name, - calculation: {module, opts}, - type: type, - constraints: constraints - } -> - {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) - expression = module.expression(opts, new_calc.context) - - expression = - Ash.Expr.fill_template( - expression, - actor: aggregate.context.actor, - tenant: aggregate.query.to_tenant, - args: %{}, - context: aggregate.context - ) - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: ecto_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, ecto_query} = - AshSql.Calculation.add_calculations( - ecto_query, - [{new_calc, expression}], - ecto_query.__ash_bindings__.resource, - source_binding, - true - ) - - ecto_query - - %Ash.Query.Calculation{ - module: module, - opts: opts, - context: context - } = calc -> - expression = module.expression(opts, context) - - expression = - Ash.Expr.fill_template( - expression, - actor: context.actor, - tenant: aggregate.query.to_tenant, - args: context.arguments, - context: context.source_context - ) - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: ecto_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, ecto_query} = - AshSql.Calculation.add_calculations( - ecto_query, - [{calc, expression}], - ecto_query.__ash_bindings__.resource, - source_binding, - true - ) - - ecto_query - - _ -> - ecto_query - end - - ref = - %Ash.Query.Ref{ - attribute: aggregate_field(aggregate, aggregate.query.resource, query), - relationship_path: [], - resource: aggregate.query.resource - } - - value = - Ecto.Query.dynamic(field(as(^query.__ash_bindings__.current), ^ref.attribute.name)) - - AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - query = - if has_parent_expr?(aggregate.query.filter) do - from(row in query, - left_lateral_join: related in subquery(ecto_query), - on: true, - as: ^query.__ash_bindings__.current - ) - else - from(row in query, - left_join: related in subquery(ecto_query), - on: true, - as: ^query.__ash_bindings__.current - ) - end - - query = - AshSql.Bindings.add_binding( - query, - %{ - path: path, - type: :aggregate, - aggregates: [aggregate] - } - ) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - with_default = - if aggregate.default_value do - if type do - type_expr = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^type_expr)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - {:ok, query, casted} - - {:error, error} -> - {:error, error} - end - end - - defp add_first_join_aggregate( - query, - resource, - aggregate, - root_data, - first_relationship, - _source_binding - ) do - {resource, path} = - case root_data do - {resource, path} -> - {resource, path} - - _ -> - {resource, []} - end - - join_filters = - if has_filter?(aggregate) do - %{(path ++ aggregate.relationship_path) => aggregate.query.filter} - else - %{} - end - - case AshSql.Join.join_all_relationships( - query, - nil, - [], - [ - {:left, - AshSql.Join.relationship_path_to_relationships( - resource, - path ++ aggregate.relationship_path - )} - ], - [], - nil, - false, - join_filters - ) do - {:ok, query} -> - ref = - aggregate_field_ref( - aggregate, - Ash.Resource.Info.related(resource, path ++ aggregate.relationship_path), - path ++ aggregate.relationship_path, - query, - first_relationship - ) - - {:ok, query} = AshSql.Join.join_all_relationships(query, ref) - - {value, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - with_default = - if aggregate.default_value do - if type do - type_expr = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^type_expr)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), casted} - - {:error, error} -> - {:error, error} - end - end - - defp maybe_filter_subquery( - agg_query, - first_relationship, - relationship_path, - aggregates, - is_single?, - source_binding - ) do - Enum.reduce_while(aggregates, {:ok, agg_query}, fn aggregate, {:ok, agg_query} -> - filter = - if !Enum.empty?(relationship_path) && aggregate.query.filter do - Ash.Filter.move_to_relationship_path( - aggregate.query.filter, - relationship_path - ) - |> Map.put(:resource, first_relationship.destination) - else - aggregate.query.filter - end - - # For unrelated aggregates (first_relationship is nil), use the aggregate's resource - # For related aggregates, use the relationship destination - related = - if first_relationship do - first_relationship.destination - else - aggregate.query.resource - end - - field = - case aggregate.field do - field when is_atom(field) -> - related - |> Ash.Resource.Info.related(relationship_path) - |> Ash.Resource.Info.field(field) - - field -> - field - end - - root_data = - case first_relationship do - nil -> - nil - - %{destination: destination, name: name} -> - {destination, [name]} - end - - agg_query = - case field do - %Ash.Query.Aggregate{} = aggregate -> - {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) - - agg_query - - %Ash.Resource.Aggregate{} = aggregate -> - {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) - - agg_query - - %Ash.Resource.Calculation{ - name: name, - calculation: {module, opts}, - type: type, - constraints: constraints - } -> - {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) - expression = module.expression(opts, new_calc.context) - - expression = - Ash.Expr.fill_template( - expression, - actor: aggregate.context.actor, - tenant: aggregate.query.to_tenant, - args: %{}, - context: aggregate.context - ) - - expression = - if Enum.empty?(relationship_path) do - expression - else - Ash.Filter.move_to_relationship_path( - expression, - relationship_path - ) - end - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: agg_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, agg_query} = - AshSql.Calculation.add_calculations( - agg_query, - [{new_calc, expression}], - agg_query.__ash_bindings__.resource, - source_binding, - false - ) - - agg_query - - %Ash.Query.Calculation{ - module: module, - opts: opts, - context: context - } = calc -> - expression = module.expression(opts, context) - - expression = - Ash.Expr.fill_template( - expression, - actor: context.actor, - tenant: aggregate.query.to_tenant, - args: context.arguments, - context: context.source_context - ) - - expression = - if Enum.empty?(relationship_path) do - expression - else - Ash.Filter.move_to_relationship_path( - expression, - relationship_path - ) - end - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: agg_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, agg_query} = - AshSql.Calculation.add_calculations( - agg_query, - [{calc, expression}], - agg_query.__ash_bindings__.resource, - source_binding, - false - ) - - agg_query - - _ -> - agg_query - end - - if has_filter?(aggregate.query) && is_single? do - {:cont, AshSql.Filter.filter(agg_query, filter, agg_query.__ash_bindings__.resource)} - else - {:cont, {:ok, agg_query}} - end - end) - end - - defp join_subquery( - query, - subquery, - nil, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: sub in subquery(subquery), - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( + is_single?, + first_relationship + ) do + AshSql.Aggregate.Lateral.add_subquery_aggregate_select( query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } + relationship_path, + aggregate, + resource, + is_single?, + first_relationship ) end - defp join_subquery( - query, - subquery, - %{manual: {_, _}}, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: sub in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( + def aggregate_field_ref(aggregate, resource, relationship_path, query, first_relationship) do + AshSql.Aggregate.Lateral.aggregate_field_ref( + aggregate, + resource, + relationship_path, query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } + first_relationship ) end - defp join_subquery( - query, - subquery, - %{type: :many_to_many}, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: agg in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - query - |> AshSql.Bindings.add_binding(%{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - }) - |> AshSql.Bindings.merge_expr_accumulator(%AshSql.Expr.ExprInfo{}) + def aggregate_field(aggregate, resource, query) do + AshSql.Aggregate.Lateral.aggregate_field(aggregate, resource, query) end - defp join_subquery( - query, - subquery, - _first_relationship, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: agg in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( - query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } - ) + def wrap_in_subquery_for_aggregates(query) do + AshSql.Aggregate.Lateral.wrap_in_subquery_for_aggregates(query) end - def next_aggregate_name(i) do - @next_aggregate_names[i] || - raise Ash.Error.Framework.AssumptionFailed, - message: """ - All 1000 static names for aggregates have been used in a single query. - Congratulations, this means that you have gone so wildly beyond our imagination - of how much can fit into a single quer. Please file an issue and we will raise the limit. - """ - end + defdelegate limit_one_first_aggregate?(resource, aggregate), to: AshSql.Aggregate.Lateral - defp select_all_aggregates( - aggregates, - joined, - relationship_path, - _query, - is_single?, - resource, - first_relationship - ) do - Enum.reduce(aggregates, joined, fn aggregate, joined -> - add_subquery_aggregate_select( - joined, - relationship_path, - aggregate, - resource, - is_single?, - first_relationship - ) - end) - end - - defp join_all_relationships( - agg_root_query, - aggregates, - relationship_path, - first_relationship, - _is_single?, - join_filters - ) do - if Enum.empty?(relationship_path) do - {:ok, agg_root_query} - else - join_filters = - Enum.reduce(join_filters, %{}, fn {key, value}, acc -> - if List.starts_with?(key, [first_relationship.name]) do - Map.put(acc, Enum.drop(key, 1), value) - else - acc - end - end) - - relationships = - first_relationship.destination - |> AshSql.Join.relationship_path_to_relationships(relationship_path) - |> List.update_at(-1, &override_read_action(&1, hd(aggregates).query.action.name)) - - AshSql.Join.join_all_relationships( - agg_root_query, - Map.values(join_filters), - [], - [ - {:inner, relationships} - ], - [], - nil, - false, - join_filters, - agg_root_query - ) - end - end - - @doc false - def can_group?(_, %{kind: :exists}, _), do: false - def can_group?(_, %{kind: :list}, _), do: false - - def can_group?(resource, aggregate, query) do - can_group_kind?(aggregate, resource, query) && !has_exists?(aggregate) && - !references_to_many_relationships?(aggregate) && - !optimizable_first_aggregate?(resource, aggregate, query) && - !limit_one_first_aggregate?(resource, aggregate) && - !has_parent_expr?(aggregate.query.filter) - end - - defp has_parent_expr?(filter, depth \\ 0) do - not is_nil( - Ash.Filter.find( - filter, - fn - %Ash.Query.Call{name: :parent, args: [expr]} -> - if depth == 0 do - true - else - has_parent_expr?(expr, depth - 1) - end - - %Ash.Query.Exists{expr: expr} -> - has_parent_expr?(expr, depth + 1) - - %Ash.Query.Parent{expr: expr} -> - if depth == 0 do - true - else - has_parent_expr?(expr, depth - 1) - end - - %Ash.Query.Ref{ - attribute: %Ash.Query.Aggregate{ - field: %Ash.Query.Calculation{module: module, opts: opts, context: context} - } - } -> - if module.has_expression?() do - Ash.Resource.Calculation.expression(module, opts, context) - |> has_parent_expr?(depth + 1) - else - false - end - - _other -> - false - end, - true, - true, - true - ) - ) - end - - # We can potentially optimize this. We don't have to prevent aggregates that reference - # relationships from joining, we can - # 1. group up the ones that do join relationships by the relationships they join - # 2. potentially group them all up that join to relationships and just join to all the relationships - # but this method is predictable and easy so we're starting by just not grouping them - defp references_to_many_relationships?(aggregate) do - if aggregate.query do - aggregate.query.filter - |> Ash.Filter.relationship_paths() - |> Enum.any?(&to_many_path?(aggregate.query.resource, &1)) - else - false - end - end - - defp to_many_path?(_resource, []), do: false - - defp to_many_path?(resource, [rel | rest]) do - case Ash.Resource.Info.relationship(resource, rel) do - %{cardinality: :many} -> - true - - nil -> - raise """ - No such relationship #{inspect(resource)}.#{rel} - """ - - rel -> - to_many_path?(rel.destination, rest) - end - end - - defp can_group_kind?(aggregate, resource, query) do - if aggregate.kind == :first do - if array_type?(resource, aggregate) || - optimizable_first_aggregate?(resource, aggregate, query) do - false - else - true - end - else - true - end - end - - @doc false - def optimizable_first_aggregate?( - resource, - %{ - kind: :first, - relationship_path: relationship_path, - join_filters: join_filters, - field: %Ash.Query.Calculation{} = field - } = aggregate, - _ - ) do - ref = - %Ash.Query.Ref{ - attribute: field, - relationship_path: relationship_path, - resource: resource - } - - with true <- join_filters == %{}, - [] <- Ash.Filter.used_aggregates(ref, :all), - [] <- Ash.Filter.relationship_paths(ref), - true <- read_action_matches_relationship_default?(resource, aggregate) do - true - else - _ -> - false - end - end - - def optimizable_first_aggregate?( - _resource, - %{ - kind: :first, - field: %Ash.Query.Aggregate{} - }, - _ - ) do - false - end - - def optimizable_first_aggregate?( - resource, - %{ - name: name, - kind: :first, - relationship_path: relationship_path, - join_filters: join_filters, - query: %{resource: related}, - field: field - } = aggregate, - query - ) do - related - |> Ash.Resource.Info.field(field) - |> case do - %Ash.Resource.Aggregate{} -> - false - - %Ash.Resource.Calculation{} -> - field = aggregate_field(aggregate, resource, query) - - ref = - %Ash.Query.Ref{ - attribute: field, - relationship_path: relationship_path, - resource: resource - } - - with [] <- Ash.Filter.used_aggregates(ref, :all), - [] <- Ash.Filter.relationship_paths(ref), - true <- read_action_matches_relationship_default?(resource, aggregate) do - true - else - _ -> - false - end - - nil -> - false - - _ -> - (name in query.__ash_bindings__.sql_behaviour.simple_join_first_aggregates(resource) || - (join_filters in [nil, %{}, []] && - single_path?(resource, relationship_path))) && - read_action_matches_relationship_default?(resource, aggregate) - end - end - - def optimizable_first_aggregate?(_, _, _), do: false - - defp array_type?(resource, aggregate) do - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - case aggregate.field do - nil -> - false - - %{type: {:array, _}} -> - true - - type when is_atom(type) -> - case Ash.Resource.Info.field(related, aggregate.field).type do - {:array, _} -> - true - - _ -> - false - end - - _ -> - false - end - end - - defp has_exists?(aggregate) do - !!Ash.Filter.find(aggregate.query && aggregate.query.filter, fn - %Ash.Query.Exists{} -> true - _ -> false - end) - end - - defp add_aggregate_selects(query, dynamics) do - {in_aggregates, in_body} = - Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) - - aggs = - in_body - |> Map.new(fn {load, _, dynamic} -> - {load, dynamic} - end) - - aggs = - if Enum.empty?(in_aggregates) do - aggs - else - Map.put( - aggs, - :aggregates, - Map.new(in_aggregates, fn {_, name, dynamic} -> - {name, dynamic} - end) - ) - end - - Ecto.Query.select_merge(query, ^aggs) - end - - defp select_dynamic(_resource, query, aggregate, binding) do - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - field = - if type do - field_ref = Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) - query.__ash_bindings__.sql_behaviour.type_expr(field_ref, type) - else - Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) - end - - coalesced = - if is_nil(aggregate.default_value) do - field - else - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic( - coalesce( - ^field, - ^typed_default - ) - ) - else - Ecto.Query.dynamic( - coalesce( - ^field, - ^aggregate.default_value - ) - ) - end - end - - if type do - query.__ash_bindings__.sql_behaviour.type_expr(coalesced, type) - else - coalesced - end - end - - defp has_filter?(nil), do: false - defp has_filter?(%{filter: nil}), do: false - defp has_filter?(%{filter: %Ash.Filter{expression: nil}}), do: false - defp has_filter?(_), do: true - - defp has_sort?(nil), do: false - defp has_sort?(%{sort: nil}), do: false - defp has_sort?(%{sort: []}), do: false - defp has_sort?(%{sort: _}), do: true - defp has_sort?(_), do: false - - def add_subquery_aggregate_select( - query, - relationship_path, - %{kind: :first} = aggregate, - resource, - is_single?, - first_relationship - ) do - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - {field, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - has_sort? = has_sort?(aggregate.query) - - array_agg = - if aggregate.include_nil? do - # any_value() ignores NULLs by design in PostgreSQL, so we must use - # array_agg with [1] indexing when include_nil? is true - "array_agg" - else - query.__ash_bindings__.sql_behaviour.list_aggregate(aggregate.query.resource) - end - - {sorted, include_nil_filter_field, query} = - if has_sort? || first_relationship.sort not in [nil, []] do - {sort, binding} = - if has_sort? do - {aggregate.query.sort, binding} - else - {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} - end - - {:ok, sort_expr, query} = - AshSql.Sort.sort( - query, - sort, - Ash.Resource.Info.related( - query.__ash_bindings__.resource, - relationship_path - ), - relationship_path, - binding, - :return - ) - - if aggregate.include_nil? do - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - ["#{array_agg}(? ORDER BY #{question_marks})", field] ++ sort_expr - ) - - {sort_expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {sort_expr, nil, query} - else - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - {expr, include_nil_filter_field} = - if has_filter?(aggregate.query) and !is_single? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "#{array_agg}(? ORDER BY #{question_marks})", - field - ] ++ - sort_expr - ) - - {expr, field} - else - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "#{array_agg}(? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, nil} - end - - {sort_expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {sort_expr, include_nil_filter_field, query} - end - else - case array_agg do - "array_agg" -> - {Ecto.Query.dynamic( - [row], - fragment("array_agg(?)", ^field) - ), nil, query} - - "any_value" -> - {Ecto.Query.dynamic( - [row], - fragment("any_value(?)", ^field) - ), nil, query} - end - end - - {query, filtered} = - filter_field( - sorted, - include_nil_filter_field, - query, - aggregate, - relationship_path, - is_single? - ) - - value = - if array_agg == "array_agg" do - Ecto.Query.dynamic(fragment("(?)[1]", ^filtered)) - else - filtered - end - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - query = AshSql.Bindings.merge_expr_accumulator(query, acc) - - select_or_merge( - query, - aggregate.name, - casted - ) - end - - def add_subquery_aggregate_select( - query, - relationship_path, - %{kind: :list} = aggregate, - resource, - is_single?, - first_relationship - ) do - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - {field, acc} = - AshSql.Expr.dynamic_expr( - query, - ref, - Map.put(query.__ash_bindings__, :location, :aggregate), - false - ) - - related = - Ash.Resource.Info.related( - query.__ash_bindings__.resource, - relationship_path - ) - - has_sort? = has_sort?(aggregate.query) - - {sorted, include_nil_filter_field, query} = - if has_sort? || (first_relationship && first_relationship.sort not in [nil, []]) do - {sort, binding} = - if has_sort? do - {aggregate.query.sort, binding} - else - {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} - end - - {:ok, sort_expr, query} = - AshSql.Sort.sort( - query, - sort, - related, - relationship_path, - binding, - :return - ) - - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - distinct = - if Map.get(aggregate, :uniq?) do - "DISTINCT " - else - "" - end - - {expr, include_nil_filter_field} = - if aggregate.include_nil? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - ["array_agg(#{distinct}? ORDER BY #{question_marks})", field] ++ sort_expr - ) - - {expr, nil} - else - if has_filter?(aggregate.query) and !is_single? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "array_agg(#{distinct}? ORDER BY #{question_marks})", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, field} - else - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "array_agg(#{distinct}? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, nil} - end - end - - {expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {expr, include_nil_filter_field, query} - else - if Map.get(aggregate, :uniq?) do - {Ecto.Query.dynamic( - [row], - fragment("array_agg(DISTINCT ?)", ^field) - ), nil, query} - else - {Ecto.Query.dynamic( - [row], - fragment("array_agg(?)", ^field) - ), nil, query} - end - end - - {query, filtered} = - filter_field( - sorted, - include_nil_filter_field, - query, - aggregate, - relationship_path, - is_single? - ) - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) - end - else - filtered - end - - cast = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - query = AshSql.Bindings.merge_expr_accumulator(query, acc) - - select_or_merge( - query, - aggregate.name, - cast - ) - end - - def add_subquery_aggregate_select( - query, - relationship_path, - %{kind: kind} = aggregate, - resource, - is_single?, - first_relationship - ) - when kind in [:count, :sum, :avg, :max, :min, :custom] do - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - {field, query} = - case kind do - :custom -> - # we won't use this if its custom so don't try to make one - {nil, query} - - :count -> - if aggregate.field do - {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} - else - {nil, query} - end - - _ -> - {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} - end - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - field = - case kind do - :count -> - cond do - !aggregate.field -> - Ecto.Query.dynamic([row], count()) - - Map.get(aggregate, :uniq?) -> - Ecto.Query.dynamic([row], count(^field, :distinct)) - - match?(%{attribute: %{allow_nil?: false}}, ref) -> - Ecto.Query.dynamic([row], count()) - - true -> - Ecto.Query.dynamic([row], count(^field)) - end - - :sum -> - Ecto.Query.dynamic([row], sum(^field)) - - :avg -> - Ecto.Query.dynamic([row], avg(^field)) - - :max -> - Ecto.Query.dynamic([row], max(^field)) - - :min -> - Ecto.Query.dynamic([row], min(^field)) - - :custom -> - {module, opts} = aggregate.implementation - - module.dynamic(opts, binding) - end - - {query, filtered} = filter_field(field, nil, query, aggregate, relationship_path, is_single?) - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) - end - else - filtered - end - - cast = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - select_or_merge(query, aggregate.name, cast) - end - - defp filter_field(field, include_nil_filter_field, query, _aggregate, _relationship_path, true) do - if include_nil_filter_field do - {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} - else - {query, field} - end - end + defdelegate resource_aggregate_to_aggregate(resource, aggregate, opts \\ []), + to: AshSql.Aggregate.Lateral - defp filter_field( - field, - include_nil_filter_field, - query, - aggregate, - relationship_path, - _is_single? - ) do - if has_filter?(aggregate.query) do - filter = - Ash.Filter.move_to_relationship_path( - aggregate.query.filter, - relationship_path - ) - - used_aggregates = Ash.Filter.used_aggregates(filter, []) - - # here we bypass an inner join. - # Really, we should check if all aggs in a group - # could do the same inner join, then do an inner join - {:ok, query} = - AshSql.Join.join_all_relationships( - query, - filter, - [], - nil, - [], - nil, - true, - nil, - nil, - true - ) - - {:ok, query} = - add_aggregates( - query, - used_aggregates, - query.__ash_bindings__.resource, - false, - query.__ash_bindings__.root_binding - ) - - {expr, acc} = - AshSql.Expr.dynamic_expr( - query, - filter, - query.__ash_bindings__, - false, - {aggregate.type, aggregate.constraints} - ) - - if include_nil_filter_field do - {AshSql.Bindings.merge_expr_accumulator(query, acc), - Ecto.Query.dynamic(filter(^field, ^expr and not is_nil(^include_nil_filter_field)))} - else - {AshSql.Bindings.merge_expr_accumulator(query, acc), - Ecto.Query.dynamic(filter(^field, ^expr))} - end - else - if include_nil_filter_field do - {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} - else - {query, field} - end - end - end - - defp select_or_merge(query, aggregate_name, casted) do - query = - if query.select do - query - else - Ecto.Query.select(query, %{}) - end - - Ecto.Query.select_merge(query, ^%{aggregate_name => casted}) - end - - def aggregate_field_ref(aggregate, resource, relationship_path, query, first_relationship) do - if aggregate.kind == :count && !aggregate.field do - nil - else - %Ash.Query.Ref{ - attribute: aggregate_field(aggregate, resource, query), - relationship_path: relationship_path, - resource: query.__ash_bindings__.resource - } - |> case do - %{attribute: %Ash.Resource.Aggregate{}} = ref when not is_nil(first_relationship) -> - if first_relationship do - %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} - else - ref - end - - %{attribute: %Ash.Query.Aggregate{}} = ref when not is_nil(first_relationship) -> - if first_relationship do - %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} - else - ref - end - - other -> - other - end - end - end - - defp read_action_matches_relationship_default?(_resource, %{relationship_path: []}), do: true - - defp read_action_matches_relationship_default?(resource, aggregate) do - last_relationship = - resource - |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) - |> List.last() - - default_action_name = - last_relationship.read_action || - Ash.Resource.Info.primary_action!(last_relationship.destination, :read).name - - case aggregate.query && aggregate.query.action do - nil -> true - action -> action.name == default_action_name - end - end - - defp override_read_action(relationship, action_name) do - if relationship.read_action == action_name do - relationship - else - relationship - |> Map.put(:read_action, action_name) - |> Map.replace(:read_action_arguments, %{}) - end - end - - defp single_path?(_, []), do: true - - defp single_path?(resource, [relationship | rest]) do - relationship = Ash.Resource.Info.relationship(resource, relationship) - - !Map.get(relationship, :from_many?) && - (relationship.type == :belongs_to || - has_one_with_identity?(relationship)) && - single_path?(relationship.destination, rest) - end - - defp has_one_with_identity?(%{type: :has_one, from_many?: false} = relationship) do - Ash.Resource.Info.primary_key(relationship.destination) == [ - relationship.destination_attribute - ] || - relationship.destination - |> Ash.Resource.Info.identities() - |> Enum.any?(fn %{keys: keys} -> - keys == [relationship.destination_attribute] - end) - end - - defp has_one_with_identity?(_), do: false - - @doc false - def aggregate_field(aggregate, resource, query) do - if is_atom(aggregate.field) do - case Ash.Resource.Info.field( - resource, - aggregate.field || List.first(Ash.Resource.Info.primary_key(resource)) - ) do - %Ash.Resource.Calculation{calculation: {module, opts}} = calculation -> - calc_type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - calculation.type, - Map.get(calculation, :constraints, []), - :calculation - ) - - AshSql.Expr.validate_type!(query, calc_type, "#{inspect(calculation.name)}") - - {:ok, query_calc} = - Ash.Query.Calculation.new( - calculation.name, - module, - opts, - calculation.type, - calculation.constraints - ) - - Ash.Actions.Read.add_calc_context( - query_calc, - aggregate.context.actor, - aggregate.context.authorize?, - aggregate.context.tenant, - aggregate.context.tracer, - query.__ash_bindings__[:domain], - aggregate.query.resource, - parent_stack: [ - query.__ash_bindings__.resource | query.__ash_bindings__[:parent_resources] || [] - ] - ) - - nil -> - raise "no such aggregate field: #{inspect(resource)}.#{aggregate.field}" - - other -> - other - end - else - aggregate.field - end - end - - def wrap_in_subquery_for_aggregates(query) do - resource = query.__ash_bindings__.resource - selected_by_default = Ash.Resource.Info.selected_by_default_attribute_names(resource) - - selected_fields = - query.__ash_bindings__[:select] || - extract_selected_fields(query, resource, selected_by_default) - - all_attr_names = - resource - |> Ash.Resource.Info.attribute_names() - |> MapSet.to_list() - - # An upgraded combination query's source only exposes the default - # attributes plus the combination fieldset; others can't be re-selected. - available_attr_names = - if query.__ash_bindings__[:subquery_upgrade?] do - MapSet.union( - selected_by_default, - MapSet.new(query.__ash_bindings__[:already_selected] || []) - ) - else - MapSet.new(all_attr_names) - end - - to_select = - Enum.reject(all_attr_names, &(&1 in selected_fields or &1 not in available_attr_names)) - - query_with_all_attrs = - case query.select do - %Ecto.Query.SelectExpr{expr: {:merge, _, [l, {:%{}, [], kw}]}} -> - put_in( - query.select.expr, - {:merge, [], - [ - l, - {:%{}, [], - kw ++ - Enum.map( - to_select, - &{&1, - {{:., [], [{:&, [], [query.__ash_bindings__.root_binding]}, &1]}, [], []}} - )} - ]} - ) - - _ -> - from(row in query, - select_merge: struct(row, ^to_select) - ) - end - - # Flatten nested calculations/aggregates maps before creating subquery - # Ecto doesn't allow nested maps in subquery select expressions - {calculations_require_rewrite, aggregates_require_rewrite, query_with_all_attrs} = - AshSql.Query.rewrite_nested_selects(query_with_all_attrs) - - # After flattening, we need to: - # 1. Use the updated select_calculations from the rewritten query (which excludes :calculations) - # 2. Add the flattened calculation/aggregate field names to the select - flattened_calc_fields = Map.keys(calculations_require_rewrite) - flattened_agg_fields = Map.keys(aggregates_require_rewrite) - - # Get select_calculations from the rewritten query (it has :calculations removed) - select_calculations = - (query_with_all_attrs.__ash_bindings__[:select_calculations] || []) -- [:calculations] - - select_aggregates = - (query_with_all_attrs.__ash_bindings__[:select_aggregates] || []) -- [:aggregates] - - reselected_fields = - Enum.concat([ - selected_fields, - select_calculations, - select_aggregates, - flattened_calc_fields, - flattened_agg_fields - ]) - - # Upgraded combination queries return maps, which `struct/2` rejects, and - # `map/2` would drop field type info — merge each field explicitly instead. - subquery_query = - if query.__ash_bindings__[:subquery_upgrade?] do - root_binding = query.__ash_bindings__.root_binding - - base = - from(row in subquery(query_with_all_attrs), - as: ^root_binding, - select: %{} - ) - - Enum.reduce(reselected_fields, base, fn field, q -> - from(row in q, select_merge: %{^field => field(as(^root_binding), ^field)}) - end) - else - from(row in subquery(query_with_all_attrs), - as: ^query.__ash_bindings__.root_binding, - select: struct(row, ^reselected_fields) - ) - end - - root_binding = query.__ash_bindings__.root_binding - - only_root_binding = %{ - root_binding => query.__ash_bindings__.bindings[root_binding] - } - - new_bindings = - query.__ash_bindings__ - |> Map.put(:bindings, only_root_binding) - |> Map.delete(:__order__?) - |> Map.update( - :calculations_require_rewrite, - calculations_require_rewrite, - &Map.merge(&1, calculations_require_rewrite) - ) - |> Map.update( - :aggregates_require_rewrite, - aggregates_require_rewrite, - &Map.merge(&1, aggregates_require_rewrite) - ) - - Map.put(subquery_query, :__ash_bindings__, new_bindings) - end - - defp extract_selected_fields( - %{select: %Ecto.Query.SelectExpr{expr: expr, take: take}}, - resource, - all_attribute_names - ) do - Enum.uniq(extract_fields_from_expr(expr, resource, take, all_attribute_names)) - end - - defp extract_fields_from_expr(expr, resource, take, all_attribute_names) do - case expr do - {:&, [], [ix]} -> - case take do - %{^ix => {:struct, fields}} when is_list(fields) -> - fields - - %{^ix => {:map, fields}} when is_list(fields) -> - fields - - take when take == %{} -> - all_attribute_names - - _ -> - [] - end - - {:%{}, [], fields} -> - Enum.map(fields, fn {field_name, _} -> field_name end) - - {:%, [], [_struct, {:%{}, [], fields}]} -> - Enum.map(fields, fn {field_name, _} -> field_name end) - - {:merge, _, [sel1, sel2]} -> - Enum.concat( - extract_fields_from_expr(sel1, resource, take, all_attribute_names), - extract_fields_from_expr(sel2, resource, take, all_attribute_names) - ) - - _other -> - all_attribute_names - end - end end diff --git a/lib/aggregate/lateral.ex b/lib/aggregate/lateral.ex new file mode 100644 index 0000000..ac77450 --- /dev/null +++ b/lib/aggregate/lateral.ex @@ -0,0 +1,2954 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Lateral do + @moduledoc false + + require Ecto.Query + import Ecto.Query, only: [from: 2, subquery: 1] + + @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> + Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") + end) + + def add_aggregates( + query, + aggregates, + resource, + select?, + source_binding, + root_data \\ nil + ) + + def add_aggregates(query, [], _, _, _, _), do: {:ok, query} + + def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do + case resource_aggregates_to_aggregates(resource, query, aggregates) do + {:ok, aggregates} -> + root_data_path = + case root_data do + {_, path} -> + path + + _ -> + [] + end + + tenant = + case Enum.at(aggregates, 0) do + %{context: %{tenant: tenant}} -> + Ash.ToTenant.to_tenant(tenant, resource) + + _ -> + nil + end + + {query, aggregates} = + Enum.reduce( + aggregates, + {query, []}, + fn aggregate, {query, aggregates} -> + if is_atom(aggregate.name) do + existing_agg = query.__ash_bindings__.aggregate_defs[aggregate.name] + + if existing_agg && different_queries?(existing_agg.query, aggregate.query) do + {query, name} = use_aggregate_name(query, aggregate.name) + {query, [%{aggregate | name: name} | aggregates]} + else + {query, [aggregate | aggregates]} + end + else + {query, name} = use_aggregate_name(query, aggregate.name) + + {query, [%{aggregate | name: name} | aggregates]} + end + end + ) + + {already_computed_aggregates, remaining_aggregates} = + aggregates + |> Enum.uniq_by(& &1.name) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, [])) + + query = + if Enum.any?(already_computed_aggregates) && select? do + query.__ash_bindings__.bindings + |> Enum.filter(fn + {_binding, %{type: :aggregate}} -> true + _ -> false + end) + |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> + q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) + + Enum.reduce(aggs, q, fn agg, q -> + if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do + q = + update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> + [agg.name | select_aggs] + end) + + if agg.default_value do + from(row in q, + select_merge: %{ + ^agg.name => + coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) + } + ) + else + from(row in q, + select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} + ) + end + else + q + end + end) + end) + else + query + end + + query = + if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && + !query.__ash_bindings__[:lateral_join?] && + Enum.any?( + remaining_aggregates, + &(not optimizable_first_aggregate?(resource, &1, query)) + ) do + wrap_in_subquery_for_aggregates(query) + else + query + end + + query = + if root_data_path == [] do + query + |> Map.update!(:__ash_bindings__, fn bindings -> + bindings + |> Map.update!(:aggregate_defs, fn aggregate_defs -> + Map.merge(aggregate_defs, Map.new(aggregates, &{&1.name, &1})) + end) + end) + else + query + end + + result = + remaining_aggregates + |> Enum.group_by(fn aggregate -> + expanded_path = + aggregate.resource + |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) + |> Enum.map(& &1.name) + + {expanded_path, aggregate.resource, aggregate.join_filters || %{}, + aggregate.query.action.name} + end) + |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> + {can_group, cant_group} = + Enum.split_with(aggregates, &can_group?(resource, &1, query)) + + [{{path, resource, join_filters, read_action}, can_group}] ++ + Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) + end) + |> Enum.reject(fn + {_, []} -> + true + + _ -> + false + end) + |> Enum.reduce_while( + {:ok, query, []}, + fn {{path, resource, join_filters, read_action}, aggregates}, + {:ok, query, dynamics} -> + related = Ash.Resource.Info.related(resource, path) + read_action = Ash.Resource.Info.action(related, read_action) + + if read_action.modify_query do + raise """ + Data layer does not currently support aggregates over read actions that use `modify_query`. + + Resource: #{inspect(resource)} + Relationship Path: #{inspect(path)} + Action: #{read_action.name} + """ + end + + {first_relationship, relationship_path} = + case path do + [] -> + {nil, []} + + [first_relationship | rest] -> + case Ash.Resource.Info.relationship(resource, first_relationship) do + nil -> + raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" + + first_relationship -> + if rest == [] do + {override_read_action(first_relationship, read_action.name), rest} + else + {first_relationship, rest} + end + end + end + + hydrated_agg_refs = + aggregates + |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) + |> Ash.Filter.hydrate_refs(%{ + resource: Enum.at(aggregates, 0).query.resource, + parent_stack: + if(first_relationship, do: [first_relationship.source], else: [resource]) + }) + |> elem(1) + + parent_expr = + if first_relationship do + first_relationship.filter + |> Ash.Filter.hydrate_refs(%{ + resource: first_relationship.destination, + parent_stack: [first_relationship.source] + }) + |> elem(1) + |> then(&[&1 | hydrated_agg_refs]) + |> AshSql.Join.parent_expr() + end + + used_aggregates = + Ash.Filter.used_aggregates(parent_expr, []) + + {:ok, query} = + AshSql.Aggregate.add_aggregates( + query, + used_aggregates, + resource, + false, + query.__ash_bindings__.root_binding + ) + + {:ok, query} = + AshSql.Join.join_all_relationships( + query, + parent_expr, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) + + is_single? = match?([_], aggregates) + + cond do + is_single? && + optimizable_first_aggregate?( + resource, + Enum.at(aggregates, 0), + query + ) -> + case add_first_join_aggregate( + query, + resource, + hd(aggregates), + root_data, + first_relationship, + source_binding + ) do + {:ok, query, dynamic} -> + query = + if select? do + select_or_merge(query, hd(aggregates).name, dynamic) + else + query + end + + {:cont, {:ok, query, dynamics}} + + {:error, error} -> + {:halt, {:error, error}} + end + + is_single? && Enum.at(aggregates, 0).kind == :exists -> + [aggregate] = aggregates + + expr = + if is_nil(Map.get(aggregate.query, :filter)) do + true + else + Map.get(aggregate.query, :filter) + end + + {exists, acc} = + AshSql.Expr.dynamic_expr( + query, + %Ash.Query.Exists{ + path: root_data_path ++ aggregate.relationship_path, + related?: aggregate.related?, + resource: aggregate.query.resource, + expr: expr + }, + query.__ash_bindings__ + ) + + {:cont, + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), + [{aggregate.load, aggregate.name, exists} | dynamics]}} + + true -> + tmp_query = + if first_relationship && first_relationship.type == :many_to_many do + put_in(query.__ash_bindings__[:lateral_join_bindings], [ + query.__ash_bindings__.current + ]) + |> AshSql.Bindings.explicitly_set_binding( + %{ + type: :left, + path: [first_relationship.join_relationship] + }, + query.__ash_bindings__.current + ) + else + query + end + + start_bindings_at = + if first_relationship && first_relationship.type == :many_to_many do + query.__ash_bindings__.current + 1 + else + query.__ash_bindings__.current + end + + case get_subquery( + resource, + aggregates, + is_single?, + first_relationship, + relationship_path, + tmp_query, + start_bindings_at, + query, + source_binding, + root_data_path, + tenant, + join_filters + ) do + {:error, error} -> + {:error, error} + + {:ok, subquery} -> + query = + join_subquery( + query, + subquery, + first_relationship, + relationship_path, + aggregates, + source_binding, + root_data_path + ) + + if select? do + new_dynamics = + Enum.map( + aggregates, + &{&1.load, &1.name, + select_dynamic( + resource, + query, + &1, + query.__ash_bindings__.current - 1 + )} + ) + + {:cont, {:ok, query, new_dynamics ++ dynamics}} + else + {:cont, {:ok, query, dynamics}} + end + end + end + end + ) + + case result do + {:ok, query, dynamics} -> + if select? do + {:ok, add_aggregate_selects(query, dynamics)} + else + {:ok, query} + end + + {:error, error} -> + {:error, error} + end + + {:error, error} -> + {:error, error} + end + end + + defp already_added?(aggregate, bindings, root_data_path) do + Enum.any?(bindings.bindings, fn + {_, %{type: :aggregate, aggregates: aggregates, path: ^root_data_path}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _other -> + false + end) + end + + defp get_subquery( + _resource, + aggregates, + is_single?, + nil, + _relationship_path, + _tmp_query, + start_bindings_at, + query, + _source_binding, + root_data_path, + tenant, + _join_filters + ) do + first_aggregate = Enum.at(aggregates, 0) + aggregate_resource = first_aggregate.query.resource + + first_aggregate.query + |> Ash.Query.set_context(%{ + data_layer: %{ + table: nil, + parent_bindings: + Map.put( + query.__ash_bindings__, + :refs_at_path, + root_data_path + ), + start_bindings_at: start_bindings_at || 0 + } + }) + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> AshSql.Join.handle_attribute_multitenancy(tenant) + |> AshSql.Join.hydrate_refs(query.__ash_bindings__.context[:private][:actor]) + |> case do + %{valid?: true} = related_query -> + case Ash.Query.data_layer_query(related_query) do + {:ok, ecto_query} -> + {:ok, Ecto.Query.exclude(ecto_query, :select)} + + {:error, error} -> + {:error, error} + end + + %{errors: errors} -> + {:error, errors} + end + |> case do + {:ok, query} -> + maybe_filter_subquery( + query, + nil, + [], + aggregates, + is_single?, + query.__ash_bindings__.root_binding + ) + + {:error, error} -> + {:error, error} + end + |> case do + {:error, error} -> + {:error, error} + + {:ok, query} -> + if is_single? and has_filter?(Enum.at(aggregates, 0).query) do + AshSql.Filter.filter( + query, + Enum.at(aggregates, 0).query.filter, + aggregate_resource + ) + else + {:ok, query} + end + |> case do + {:error, error} -> + {:error, error} + + {:ok, filtered} -> + filtered = + AshSql.Join.set_join_prefix( + filtered, + %{query | prefix: tenant}, + aggregate_resource + ) + + {:ok, + select_all_aggregates( + aggregates, + filtered, + [], + query, + is_single?, + aggregate_resource, + nil + )} + end + end + end + + defp get_subquery( + resource, + aggregates, + is_single?, + first_relationship, + relationship_path, + tmp_query, + start_bindings_at, + query, + source_binding, + root_data_path, + tenant, + join_filters + ) do + limited? = limited_relationship?(first_relationship) + + limit_one_first? = + is_single? && relationship_path == [] && + limit_one_first_aggregate?(resource, Enum.at(aggregates, 0)) + + AshSql.Join.related_subquery( + first_relationship, + tmp_query, + start_bindings_at: start_bindings_at, + refs_at_path: root_data_path, + skip_distinct_for_first_rel?: true, + sort?: limited?, + on_subquery: fn subquery -> + base_binding = subquery.__ash_bindings__.root_binding + current_binding = subquery.__ash_bindings__.current + + subquery = + cond do + limited? -> + apply_limited_relationship_subquery( + subquery, + first_relationship, + source_binding, + base_binding + ) + + limit_one_first? -> + apply_limit_one_first_subquery( + subquery, + Enum.at(aggregates, 0), + first_relationship, + source_binding, + base_binding + ) + + true -> + subquery + |> Ecto.Query.exclude(:select) + |> Ecto.Query.select(%{}) + |> apply_relationship_subquery( + first_relationship, + query, + tenant, + source_binding, + current_binding, + base_binding + ) + end + + subquery = + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + first_relationship.destination + ) + + {:ok, subquery, _} = + apply_first_relationship_join_filters( + subquery, + query, + %AshSql.Expr.ExprInfo{}, + first_relationship, + join_filters + ) + + subquery = + set_in_group( + subquery, + query, + resource + ) + + {:ok, joined} = + join_all_relationships( + subquery, + aggregates, + relationship_path, + first_relationship, + is_single?, + join_filters + ) + + # When the aggregate's filter has already been pushed into the + # `LIMIT 1` inner subquery, it must not be applied again here. + {:ok, filtered} = + maybe_filter_subquery( + joined, + first_relationship, + relationship_path, + aggregates, + is_single? && !limit_one_first?, + subquery.__ash_bindings__.root_binding + ) + + select_all_aggregates( + aggregates, + filtered, + relationship_path, + query, + is_single?, + Ash.Resource.Info.related( + first_relationship.destination, + relationship_path + ), + first_relationship + ) + end + ) + end + + # Relationships that declare a `limit` (or `offset`) need the limit applied + # to the correlated rows *before* the aggregation's `GROUP BY`, otherwise the + # limit caps the number of groups (always 1 in a lateral join) instead of the + # number of rows per group. + defp limited_relationship?(relationship) do + (is_integer(Map.get(relationship, :limit)) or + (Map.get(relationship, :offset) || 0) > 0) and + is_nil(Map.get(relationship, :manual)) and + !Map.get(relationship, :no_attributes?) and + relationship.type != :many_to_many + end + + # Builds: + # + # SELECT ... FROM ( + # SELECT * FROM destination + # WHERE destination.destination_attribute = parent.source_attribute + # ORDER BY LIMIT + # ) AS + # GROUP BY destination_attribute + # + # The correlation, sort and limit all live in the inner subquery so that the + # limit bounds the rows per parent, and the aggregate functions fold the + # already-limited rows. + defp apply_limited_relationship_subquery(subquery, rel, source_binding, base_binding) do + field = rel.destination_attribute + + inner = + from(row in subquery, + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + as(^base_binding), + ^rel.destination_attribute + ) + ) + + inner = + case Map.get(rel, :limit) do + limit when is_integer(limit) -> Ecto.Query.limit(inner, ^limit) + _ -> inner + end + + from(row in subquery(inner), as: ^base_binding) + |> Map.put(:__ash_bindings__, subquery.__ash_bindings__) + |> Ecto.Query.select(%{}) + |> then(fn wrapped -> + from(row in wrapped, + group_by: field(row, ^field), + select_merge: %{^field => field(row, ^field)} + ) + end) + end + + @doc false + def limit_one_first_aggregate?( + resource, + %{ + kind: :first, + relationship_path: [relationship_name], + field: field, + join_filters: join_filters + } = aggregate + ) + when is_atom(field) and not is_nil(field) do + case Ash.Resource.Info.relationship(resource, relationship_name) do + nil -> + false + + relationship -> + relationship.type != :many_to_many && + is_nil(Map.get(relationship, :manual)) && + !Map.get(relationship, :no_attributes?) && + !limited_relationship?(relationship) && + join_filters in [nil, %{}, []] && + (has_sort?(aggregate.query) || relationship.sort not in [nil, []]) && + match?( + %Ash.Resource.Attribute{}, + Ash.Resource.Info.attribute(relationship.destination, field) + ) + end + end + + def limit_one_first_aggregate?(_resource, _aggregate), do: false + + defp apply_limit_one_first_subquery( + subquery, + aggregate, + rel, + source_binding, + base_binding + ) do + group_field = rel.destination_attribute + + inner = + from(row in subquery, + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + as(^base_binding), + ^rel.destination_attribute + ) + ) + + {:ok, inner} = + if has_filter?(aggregate.query) do + AshSql.Filter.filter(inner, aggregate.query.filter, rel.destination) + else + {:ok, inner} + end + + inner = + if aggregate.include_nil? do + inner + else + from(row in inner, + where: not is_nil(field(as(^base_binding), ^aggregate.field)) + ) + end + + sort = + if has_sort?(aggregate.query) do + aggregate.query.sort + else + List.wrap(rel.sort) + end + + {:ok, inner} = + AshSql.Sort.sort(inner, sort, rel.destination, [], base_binding, :direct) + + inner = Ecto.Query.limit(inner, 1) + + from(row in subquery(inner), as: ^base_binding) + |> Map.put(:__ash_bindings__, subquery.__ash_bindings__) + |> Ecto.Query.select(%{}) + |> then(fn wrapped -> + from(row in wrapped, + group_by: field(row, ^group_field), + select_merge: %{^group_field => field(row, ^group_field)} + ) + end) + end + + defp apply_relationship_subquery( + subquery, + %{manual: {module, opts}} = rel, + query, + tenant, + source_binding, + current_binding, + _base_binding + ) do + field = rel.destination_attribute + + from(row in subquery, + group_by: field(row, ^field), + select_merge: %{^field => field(row, ^field)} + ) + + subquery = + from(row in subquery, distinct: true) + + {:ok, subquery} = + apply( + module, + query.__ash_bindings__.sql_behaviour.manual_relationship_subquery_function(), + [ + opts, + source_binding, + current_binding - 1, + subquery + ] + ) + + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + rel.destination + ) + end + + defp apply_relationship_subquery( + subquery, + %{no_attributes?: true}, + _query, + _tenant, + _source_binding, + _current_binding, + _base_binding + ) do + subquery + end + + defp apply_relationship_subquery( + subquery, + %{type: :many_to_many} = rel, + query, + tenant, + source_binding, + _current_binding, + _base_binding + ) do + join_relationship_struct = + Ash.Resource.Info.relationship( + rel.source, + rel.join_relationship + ) + + {:ok, through} = + AshSql.Join.related_subquery( + join_relationship_struct, + query + ) + + field = rel.source_attribute_on_join_resource + + subquery = + from(sub in subquery, + join: through in ^through, + as: ^query.__ash_bindings__.current, + on: + field( + through, + ^rel.destination_attribute_on_join_resource + ) == + field(sub, ^rel.destination_attribute), + select_merge: map(through, ^[field]), + group_by: + field( + through, + ^rel.source_attribute_on_join_resource + ), + distinct: + field( + through, + ^rel.source_attribute_on_join_resource + ), + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + through, + ^rel.source_attribute_on_join_resource + ) + ) + + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + rel.destination + ) + end + + defp apply_relationship_subquery( + subquery, + rel, + _query, + _tenant, + source_binding, + _current_binding, + base_binding + ) do + field = rel.destination_attribute + + from(row in subquery, + group_by: field(row, ^field), + select_merge: %{^field => field(row, ^field)}, + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + as(^base_binding), + ^rel.destination_attribute + ) + ) + end + + defp set_in_group(%{__ash_bindings__: _} = query, _, _resource) do + Map.update!( + query, + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp set_in_group(%Ecto.SubQuery{} = subquery, query, resource) do + subquery = from(row in subquery, []) + + subquery + |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) + |> Map.update!( + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp set_in_group(other, query, resource) do + from(row in other, as: ^0) + |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) + |> Map.update!( + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp different_queries?(nil, nil), do: false + defp different_queries?(nil, _), do: true + defp different_queries?(_, nil), do: true + + defp different_queries?(query1, query2) do + query1.filter != query2.filter || query1.sort != query2.sort + end + + @doc false + def extract_shared_filters(aggregates) do + aggregates + |> Enum.reduce_while({nil, []}, fn + %{query: %{filter: filter}} = agg, {global_filters, aggs} when not is_nil(filter) -> + and_statements = + AshSql.Expr.split_statements(filter, :and) + + global_filters = + if global_filters do + Enum.filter(global_filters, &(&1 in and_statements)) + else + and_statements + end + + {:cont, {global_filters, [{agg, and_statements} | aggs]}} + + _, _ -> + {:halt, {:error, aggregates}} + end) + |> case do + {:error, aggregates} -> + {:error, aggregates} + + {[], _} -> + {:error, aggregates} + + {nil, _} -> + {:error, aggregates} + + {global_filters, aggregates} -> + global_filter = and_filters(Enum.uniq(global_filters)) + + aggregates = + Enum.map(aggregates, fn {agg, and_statements} -> + applicable_and_statements = + and_statements + |> Enum.reject(&(&1 in global_filters)) + |> and_filters() + + %{agg | query: %{agg.query | filter: applicable_and_statements}} + end) + + {{:ok, global_filter}, aggregates} + end + end + + defp and_filters(filters) do + Enum.reduce(filters, nil, fn expr, acc -> + if is_nil(acc) do + expr + else + Ash.Query.BooleanExpression.new(:and, expr, acc) + end + end) + end + + defp apply_first_relationship_join_filters( + agg_root_query, + query, + acc, + first_relationship, + join_filters + ) do + case join_filters[[first_relationship.name]] do + nil -> + {:ok, agg_root_query, acc} + + filter -> + with {:ok, agg_root_query} <- + AshSql.Join.join_all_relationships(agg_root_query, filter) do + agg_root_query = + AshSql.Expr.set_parent_path( + agg_root_query, + query + ) + + {query, acc} = + AshSql.Join.maybe_apply_filter( + agg_root_query, + agg_root_query, + agg_root_query.__ash_bindings__, + filter + ) + + {:ok, query, acc} + end + end + end + + defp use_aggregate_name(query, aggregate_name) do + {%{ + query + | __ash_bindings__: %{ + query.__ash_bindings__ + | current_aggregate_name: + next_aggregate_name(query.__ash_bindings__.current_aggregate_name), + aggregate_names: + Map.put( + query.__ash_bindings__.aggregate_names, + aggregate_name, + query.__ash_bindings__.current_aggregate_name + ) + } + }, query.__ash_bindings__.current_aggregate_name} + end + + defp resource_aggregates_to_aggregates(resource, query, aggregates) do + private_context = query.__ash_bindings__.context[:private] + + Enum.reduce_while(aggregates, {:ok, []}, fn + %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> + aggregate = + Ash.Actions.Read.add_calc_context( + aggregate, + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + aggregate, {:ok, aggregates} -> + resource + |> resource_aggregate_to_aggregate(aggregate, + actor: private_context[:actor], + tenant: private_context[:tenant] + ) + |> case do + {:ok, aggregate} -> + aggregate = + aggregate + |> Map.put(:load, aggregate.name) + |> Ash.Actions.Read.add_calc_context( + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + + @doc false + def resource_aggregate_to_aggregate(resource, aggregate, opts \\ []) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + read_action = + aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name + + with %{valid?: true} = aggregate_query <- + Ash.Query.for_read(related, read_action, %{}, + actor: opts[:actor], + tenant: opts[:tenant] + ), + %{valid?: true} = aggregate_query <- + Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do + Ash.Query.Aggregate.new( + resource, + aggregate.name, + aggregate.kind, + path: aggregate.relationship_path, + query: aggregate_query, + field: aggregate.field, + default: aggregate.default, + filterable?: aggregate.filterable?, + type: aggregate.type, + sortable?: aggregate.filterable?, + include_nil?: aggregate.include_nil?, + constraints: aggregate.constraints, + implementation: aggregate.implementation, + uniq?: aggregate.uniq?, + read_action: read_action, + authorize?: aggregate.authorize? + ) + else + %{errors: errors} -> + {:error, errors} + end + end + + defp add_first_join_aggregate( + query, + _resource, + %{related?: false} = aggregate, + root_data, + _, + source_binding + ) do + path = + case root_data do + {_resource, path} -> + path + + _ -> + [] + end + + subquery_result = + aggregate.query + |> Ash.Query.set_context(%{ + data_layer: %{ + table: nil, + parent_bindings: + Map.put( + query.__ash_bindings__, + :refs_at_path, + path + ), + start_bindings_at: (query.__ash_bindings__.current || 0) + 1 + } + }) + |> Ash.Query.limit(1) + |> Ash.Query.data_layer_query() + + case subquery_result do + {:ok, ecto_query} -> + ref = + %Ash.Query.Ref{ + attribute: aggregate.field, + resource: aggregate.query.resource + } + + {:ok, ecto_query} = AshSql.Join.join_all_relationships(ecto_query, ref) + + ecto_query = + case aggregate.field do + %Ash.Query.Aggregate{} = aggregate -> + {:ok, ecto_query} = + add_aggregates( + ecto_query, + [aggregate], + aggregate.query.resource, + true, + source_binding, + root_data + ) + + ecto_query + + %Ash.Resource.Aggregate{} = aggregate -> + {:ok, ecto_query} = + add_aggregates( + ecto_query, + [aggregate], + Ash.Resource.Info.related(aggregate.resource, aggregate.relationship_path), + true, + source_binding, + root_data + ) + + ecto_query + + %Ash.Resource.Calculation{ + name: name, + calculation: {module, opts}, + type: type, + constraints: constraints + } -> + {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) + expression = module.expression(opts, new_calc.context) + + expression = + Ash.Expr.fill_template( + expression, + actor: aggregate.context.actor, + tenant: aggregate.query.to_tenant, + args: %{}, + context: aggregate.context + ) + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: ecto_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, ecto_query} = + AshSql.Calculation.add_calculations( + ecto_query, + [{new_calc, expression}], + ecto_query.__ash_bindings__.resource, + source_binding, + true + ) + + ecto_query + + %Ash.Query.Calculation{ + module: module, + opts: opts, + context: context + } = calc -> + expression = module.expression(opts, context) + + expression = + Ash.Expr.fill_template( + expression, + actor: context.actor, + tenant: aggregate.query.to_tenant, + args: context.arguments, + context: context.source_context + ) + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: ecto_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, ecto_query} = + AshSql.Calculation.add_calculations( + ecto_query, + [{calc, expression}], + ecto_query.__ash_bindings__.resource, + source_binding, + true + ) + + ecto_query + + _ -> + ecto_query + end + + ref = + %Ash.Query.Ref{ + attribute: aggregate_field(aggregate, aggregate.query.resource, query), + relationship_path: [], + resource: aggregate.query.resource + } + + value = + Ecto.Query.dynamic(field(as(^query.__ash_bindings__.current), ^ref.attribute.name)) + + AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + query = + if has_parent_expr?(aggregate.query.filter) do + from(row in query, + left_lateral_join: related in subquery(ecto_query), + on: true, + as: ^query.__ash_bindings__.current + ) + else + from(row in query, + left_join: related in subquery(ecto_query), + on: true, + as: ^query.__ash_bindings__.current + ) + end + + query = + AshSql.Bindings.add_binding( + query, + %{ + path: path, + type: :aggregate, + aggregates: [aggregate] + } + ) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + with_default = + if aggregate.default_value do + if type do + type_expr = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^type_expr)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + {:ok, query, casted} + + {:error, error} -> + {:error, error} + end + end + + defp add_first_join_aggregate( + query, + resource, + aggregate, + root_data, + first_relationship, + _source_binding + ) do + {resource, path} = + case root_data do + {resource, path} -> + {resource, path} + + _ -> + {resource, []} + end + + join_filters = + if has_filter?(aggregate) do + %{(path ++ aggregate.relationship_path) => aggregate.query.filter} + else + %{} + end + + case AshSql.Join.join_all_relationships( + query, + nil, + [], + [ + {:left, + AshSql.Join.relationship_path_to_relationships( + resource, + path ++ aggregate.relationship_path + )} + ], + [], + nil, + false, + join_filters + ) do + {:ok, query} -> + ref = + aggregate_field_ref( + aggregate, + Ash.Resource.Info.related(resource, path ++ aggregate.relationship_path), + path ++ aggregate.relationship_path, + query, + first_relationship + ) + + {:ok, query} = AshSql.Join.join_all_relationships(query, ref) + + {value, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + with_default = + if aggregate.default_value do + if type do + type_expr = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^type_expr)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), casted} + + {:error, error} -> + {:error, error} + end + end + + defp maybe_filter_subquery( + agg_query, + first_relationship, + relationship_path, + aggregates, + is_single?, + source_binding + ) do + Enum.reduce_while(aggregates, {:ok, agg_query}, fn aggregate, {:ok, agg_query} -> + filter = + if !Enum.empty?(relationship_path) && aggregate.query.filter do + Ash.Filter.move_to_relationship_path( + aggregate.query.filter, + relationship_path + ) + |> Map.put(:resource, first_relationship.destination) + else + aggregate.query.filter + end + + # For unrelated aggregates (first_relationship is nil), use the aggregate's resource + # For related aggregates, use the relationship destination + related = + if first_relationship do + first_relationship.destination + else + aggregate.query.resource + end + + field = + case aggregate.field do + field when is_atom(field) -> + related + |> Ash.Resource.Info.related(relationship_path) + |> Ash.Resource.Info.field(field) + + field -> + field + end + + root_data = + case first_relationship do + nil -> + nil + + %{destination: destination, name: name} -> + {destination, [name]} + end + + agg_query = + case field do + %Ash.Query.Aggregate{} = aggregate -> + {:ok, agg_query} = + add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + + agg_query + + %Ash.Resource.Aggregate{} = aggregate -> + {:ok, agg_query} = + add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + + agg_query + + %Ash.Resource.Calculation{ + name: name, + calculation: {module, opts}, + type: type, + constraints: constraints + } -> + {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) + expression = module.expression(opts, new_calc.context) + + expression = + Ash.Expr.fill_template( + expression, + actor: aggregate.context.actor, + tenant: aggregate.query.to_tenant, + args: %{}, + context: aggregate.context + ) + + expression = + if Enum.empty?(relationship_path) do + expression + else + Ash.Filter.move_to_relationship_path( + expression, + relationship_path + ) + end + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: agg_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, agg_query} = + AshSql.Calculation.add_calculations( + agg_query, + [{new_calc, expression}], + agg_query.__ash_bindings__.resource, + source_binding, + false + ) + + agg_query + + %Ash.Query.Calculation{ + module: module, + opts: opts, + context: context + } = calc -> + expression = module.expression(opts, context) + + expression = + Ash.Expr.fill_template( + expression, + actor: context.actor, + tenant: aggregate.query.to_tenant, + args: context.arguments, + context: context.source_context + ) + + expression = + if Enum.empty?(relationship_path) do + expression + else + Ash.Filter.move_to_relationship_path( + expression, + relationship_path + ) + end + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: agg_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, agg_query} = + AshSql.Calculation.add_calculations( + agg_query, + [{calc, expression}], + agg_query.__ash_bindings__.resource, + source_binding, + false + ) + + agg_query + + _ -> + agg_query + end + + if has_filter?(aggregate.query) && is_single? do + {:cont, AshSql.Filter.filter(agg_query, filter, agg_query.__ash_bindings__.resource)} + else + {:cont, {:ok, agg_query}} + end + end) + end + + defp join_subquery( + query, + subquery, + nil, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: sub in subquery(subquery), + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + defp join_subquery( + query, + subquery, + %{manual: {_, _}}, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: sub in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + defp join_subquery( + query, + subquery, + %{type: :many_to_many}, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: agg in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + query + |> AshSql.Bindings.add_binding(%{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + }) + |> AshSql.Bindings.merge_expr_accumulator(%AshSql.Expr.ExprInfo{}) + end + + defp join_subquery( + query, + subquery, + _first_relationship, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: agg in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + def next_aggregate_name(i) do + @next_aggregate_names[i] || + raise Ash.Error.Framework.AssumptionFailed, + message: """ + All 1000 static names for aggregates have been used in a single query. + Congratulations, this means that you have gone so wildly beyond our imagination + of how much can fit into a single quer. Please file an issue and we will raise the limit. + """ + end + + defp select_all_aggregates( + aggregates, + joined, + relationship_path, + _query, + is_single?, + resource, + first_relationship + ) do + Enum.reduce(aggregates, joined, fn aggregate, joined -> + add_subquery_aggregate_select( + joined, + relationship_path, + aggregate, + resource, + is_single?, + first_relationship + ) + end) + end + + defp join_all_relationships( + agg_root_query, + aggregates, + relationship_path, + first_relationship, + _is_single?, + join_filters + ) do + if Enum.empty?(relationship_path) do + {:ok, agg_root_query} + else + join_filters = + Enum.reduce(join_filters, %{}, fn {key, value}, acc -> + if List.starts_with?(key, [first_relationship.name]) do + Map.put(acc, Enum.drop(key, 1), value) + else + acc + end + end) + + relationships = + first_relationship.destination + |> AshSql.Join.relationship_path_to_relationships(relationship_path) + |> List.update_at(-1, &override_read_action(&1, hd(aggregates).query.action.name)) + + AshSql.Join.join_all_relationships( + agg_root_query, + Map.values(join_filters), + [], + [ + {:inner, relationships} + ], + [], + nil, + false, + join_filters, + agg_root_query + ) + end + end + + @doc false + def can_group?(_, %{kind: :exists}, _), do: false + def can_group?(_, %{kind: :list}, _), do: false + + def can_group?(resource, aggregate, query) do + can_group_kind?(aggregate, resource, query) && !has_exists?(aggregate) && + !references_to_many_relationships?(aggregate) && + !optimizable_first_aggregate?(resource, aggregate, query) && + !limit_one_first_aggregate?(resource, aggregate) && + !has_parent_expr?(aggregate.query.filter) + end + + defp has_parent_expr?(filter, depth \\ 0) do + not is_nil( + Ash.Filter.find( + filter, + fn + %Ash.Query.Call{name: :parent, args: [expr]} -> + if depth == 0 do + true + else + has_parent_expr?(expr, depth - 1) + end + + %Ash.Query.Exists{expr: expr} -> + has_parent_expr?(expr, depth + 1) + + %Ash.Query.Parent{expr: expr} -> + if depth == 0 do + true + else + has_parent_expr?(expr, depth - 1) + end + + %Ash.Query.Ref{ + attribute: %Ash.Query.Aggregate{ + field: %Ash.Query.Calculation{module: module, opts: opts, context: context} + } + } -> + if module.has_expression?() do + Ash.Resource.Calculation.expression(module, opts, context) + |> has_parent_expr?(depth + 1) + else + false + end + + _other -> + false + end, + true, + true, + true + ) + ) + end + + # We can potentially optimize this. We don't have to prevent aggregates that reference + # relationships from joining, we can + # 1. group up the ones that do join relationships by the relationships they join + # 2. potentially group them all up that join to relationships and just join to all the relationships + # but this method is predictable and easy so we're starting by just not grouping them + defp references_to_many_relationships?(aggregate) do + if aggregate.query do + aggregate.query.filter + |> Ash.Filter.relationship_paths() + |> Enum.any?(&to_many_path?(aggregate.query.resource, &1)) + else + false + end + end + + defp to_many_path?(_resource, []), do: false + + defp to_many_path?(resource, [rel | rest]) do + case Ash.Resource.Info.relationship(resource, rel) do + %{cardinality: :many} -> + true + + nil -> + raise """ + No such relationship #{inspect(resource)}.#{rel} + """ + + rel -> + to_many_path?(rel.destination, rest) + end + end + + defp can_group_kind?(aggregate, resource, query) do + if aggregate.kind == :first do + if array_type?(resource, aggregate) || + optimizable_first_aggregate?(resource, aggregate, query) do + false + else + true + end + else + true + end + end + + @doc false + def optimizable_first_aggregate?( + resource, + %{ + kind: :first, + relationship_path: relationship_path, + join_filters: join_filters, + field: %Ash.Query.Calculation{} = field + } = aggregate, + _ + ) do + ref = + %Ash.Query.Ref{ + attribute: field, + relationship_path: relationship_path, + resource: resource + } + + with true <- join_filters == %{}, + [] <- Ash.Filter.used_aggregates(ref, :all), + [] <- Ash.Filter.relationship_paths(ref), + true <- read_action_matches_relationship_default?(resource, aggregate) do + true + else + _ -> + false + end + end + + def optimizable_first_aggregate?( + _resource, + %{ + kind: :first, + field: %Ash.Query.Aggregate{} + }, + _ + ) do + false + end + + def optimizable_first_aggregate?( + resource, + %{ + name: name, + kind: :first, + relationship_path: relationship_path, + join_filters: join_filters, + query: %{resource: related}, + field: field + } = aggregate, + query + ) do + related + |> Ash.Resource.Info.field(field) + |> case do + %Ash.Resource.Aggregate{} -> + false + + %Ash.Resource.Calculation{} -> + field = aggregate_field(aggregate, resource, query) + + ref = + %Ash.Query.Ref{ + attribute: field, + relationship_path: relationship_path, + resource: resource + } + + with [] <- Ash.Filter.used_aggregates(ref, :all), + [] <- Ash.Filter.relationship_paths(ref), + true <- read_action_matches_relationship_default?(resource, aggregate) do + true + else + _ -> + false + end + + nil -> + false + + _ -> + (name in query.__ash_bindings__.sql_behaviour.simple_join_first_aggregates(resource) || + (join_filters in [nil, %{}, []] && + single_path?(resource, relationship_path))) && + read_action_matches_relationship_default?(resource, aggregate) + end + end + + def optimizable_first_aggregate?(_, _, _), do: false + + defp array_type?(resource, aggregate) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + case aggregate.field do + nil -> + false + + %{type: {:array, _}} -> + true + + type when is_atom(type) -> + case Ash.Resource.Info.field(related, aggregate.field).type do + {:array, _} -> + true + + _ -> + false + end + + _ -> + false + end + end + + defp has_exists?(aggregate) do + !!Ash.Filter.find(aggregate.query && aggregate.query.filter, fn + %Ash.Query.Exists{} -> true + _ -> false + end) + end + + defp add_aggregate_selects(query, dynamics) do + {in_aggregates, in_body} = + Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) + + aggs = + in_body + |> Map.new(fn {load, _, dynamic} -> + {load, dynamic} + end) + + aggs = + if Enum.empty?(in_aggregates) do + aggs + else + Map.put( + aggs, + :aggregates, + Map.new(in_aggregates, fn {_, name, dynamic} -> + {name, dynamic} + end) + ) + end + + Ecto.Query.select_merge(query, ^aggs) + end + + defp select_dynamic(_resource, query, aggregate, binding) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + field = + if type do + field_ref = Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + query.__ash_bindings__.sql_behaviour.type_expr(field_ref, type) + else + Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + end + + coalesced = + if is_nil(aggregate.default_value) do + field + else + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic( + coalesce( + ^field, + ^typed_default + ) + ) + else + Ecto.Query.dynamic( + coalesce( + ^field, + ^aggregate.default_value + ) + ) + end + end + + if type do + query.__ash_bindings__.sql_behaviour.type_expr(coalesced, type) + else + coalesced + end + end + + defp has_filter?(nil), do: false + defp has_filter?(%{filter: nil}), do: false + defp has_filter?(%{filter: %Ash.Filter{expression: nil}}), do: false + defp has_filter?(_), do: true + + defp has_sort?(nil), do: false + defp has_sort?(%{sort: nil}), do: false + defp has_sort?(%{sort: []}), do: false + defp has_sort?(%{sort: _}), do: true + defp has_sort?(_), do: false + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: :first} = aggregate, + resource, + is_single?, + first_relationship + ) do + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + {field, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + has_sort? = has_sort?(aggregate.query) + + array_agg = + if aggregate.include_nil? do + # any_value() ignores NULLs by design in PostgreSQL, so we must use + # array_agg with [1] indexing when include_nil? is true + "array_agg" + else + query.__ash_bindings__.sql_behaviour.list_aggregate(aggregate.query.resource) + end + + {sorted, include_nil_filter_field, query} = + if has_sort? || first_relationship.sort not in [nil, []] do + {sort, binding} = + if has_sort? do + {aggregate.query.sort, binding} + else + {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} + end + + {:ok, sort_expr, query} = + AshSql.Sort.sort( + query, + sort, + Ash.Resource.Info.related( + query.__ash_bindings__.resource, + relationship_path + ), + relationship_path, + binding, + :return + ) + + if aggregate.include_nil? do + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + ["#{array_agg}(? ORDER BY #{question_marks})", field] ++ sort_expr + ) + + {sort_expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {sort_expr, nil, query} + else + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + {expr, include_nil_filter_field} = + if has_filter?(aggregate.query) and !is_single? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "#{array_agg}(? ORDER BY #{question_marks})", + field + ] ++ + sort_expr + ) + + {expr, field} + else + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "#{array_agg}(? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, nil} + end + + {sort_expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {sort_expr, include_nil_filter_field, query} + end + else + case array_agg do + "array_agg" -> + {Ecto.Query.dynamic( + [row], + fragment("array_agg(?)", ^field) + ), nil, query} + + "any_value" -> + {Ecto.Query.dynamic( + [row], + fragment("any_value(?)", ^field) + ), nil, query} + end + end + + {query, filtered} = + filter_field( + sorted, + include_nil_filter_field, + query, + aggregate, + relationship_path, + is_single? + ) + + value = + if array_agg == "array_agg" do + Ecto.Query.dynamic(fragment("(?)[1]", ^filtered)) + else + filtered + end + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + query = AshSql.Bindings.merge_expr_accumulator(query, acc) + + select_or_merge( + query, + aggregate.name, + casted + ) + end + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: :list} = aggregate, + resource, + is_single?, + first_relationship + ) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + {field, acc} = + AshSql.Expr.dynamic_expr( + query, + ref, + Map.put(query.__ash_bindings__, :location, :aggregate), + false + ) + + related = + Ash.Resource.Info.related( + query.__ash_bindings__.resource, + relationship_path + ) + + has_sort? = has_sort?(aggregate.query) + + {sorted, include_nil_filter_field, query} = + if has_sort? || (first_relationship && first_relationship.sort not in [nil, []]) do + {sort, binding} = + if has_sort? do + {aggregate.query.sort, binding} + else + {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} + end + + {:ok, sort_expr, query} = + AshSql.Sort.sort( + query, + sort, + related, + relationship_path, + binding, + :return + ) + + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + distinct = + if Map.get(aggregate, :uniq?) do + "DISTINCT " + else + "" + end + + {expr, include_nil_filter_field} = + if aggregate.include_nil? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + ["array_agg(#{distinct}? ORDER BY #{question_marks})", field] ++ sort_expr + ) + + {expr, nil} + else + if has_filter?(aggregate.query) and !is_single? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "array_agg(#{distinct}? ORDER BY #{question_marks})", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, field} + else + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "array_agg(#{distinct}? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, nil} + end + end + + {expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {expr, include_nil_filter_field, query} + else + if Map.get(aggregate, :uniq?) do + {Ecto.Query.dynamic( + [row], + fragment("array_agg(DISTINCT ?)", ^field) + ), nil, query} + else + {Ecto.Query.dynamic( + [row], + fragment("array_agg(?)", ^field) + ), nil, query} + end + end + + {query, filtered} = + filter_field( + sorted, + include_nil_filter_field, + query, + aggregate, + relationship_path, + is_single? + ) + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) + end + else + filtered + end + + cast = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + query = AshSql.Bindings.merge_expr_accumulator(query, acc) + + select_or_merge( + query, + aggregate.name, + cast + ) + end + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: kind} = aggregate, + resource, + is_single?, + first_relationship + ) + when kind in [:count, :sum, :avg, :max, :min, :custom] do + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + {field, query} = + case kind do + :custom -> + # we won't use this if its custom so don't try to make one + {nil, query} + + :count -> + if aggregate.field do + {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} + else + {nil, query} + end + + _ -> + {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} + end + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + field = + case kind do + :count -> + cond do + !aggregate.field -> + Ecto.Query.dynamic([row], count()) + + Map.get(aggregate, :uniq?) -> + Ecto.Query.dynamic([row], count(^field, :distinct)) + + match?(%{attribute: %{allow_nil?: false}}, ref) -> + Ecto.Query.dynamic([row], count()) + + true -> + Ecto.Query.dynamic([row], count(^field)) + end + + :sum -> + Ecto.Query.dynamic([row], sum(^field)) + + :avg -> + Ecto.Query.dynamic([row], avg(^field)) + + :max -> + Ecto.Query.dynamic([row], max(^field)) + + :min -> + Ecto.Query.dynamic([row], min(^field)) + + :custom -> + {module, opts} = aggregate.implementation + + module.dynamic(opts, binding) + end + + {query, filtered} = filter_field(field, nil, query, aggregate, relationship_path, is_single?) + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) + end + else + filtered + end + + cast = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + select_or_merge(query, aggregate.name, cast) + end + + defp filter_field(field, include_nil_filter_field, query, _aggregate, _relationship_path, true) do + if include_nil_filter_field do + {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} + else + {query, field} + end + end + + defp filter_field( + field, + include_nil_filter_field, + query, + aggregate, + relationship_path, + _is_single? + ) do + if has_filter?(aggregate.query) do + filter = + Ash.Filter.move_to_relationship_path( + aggregate.query.filter, + relationship_path + ) + + used_aggregates = Ash.Filter.used_aggregates(filter, []) + + # here we bypass an inner join. + # Really, we should check if all aggs in a group + # could do the same inner join, then do an inner join + {:ok, query} = + AshSql.Join.join_all_relationships( + query, + filter, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) + + {:ok, query} = + add_aggregates( + query, + used_aggregates, + query.__ash_bindings__.resource, + false, + query.__ash_bindings__.root_binding + ) + + {expr, acc} = + AshSql.Expr.dynamic_expr( + query, + filter, + query.__ash_bindings__, + false, + {aggregate.type, aggregate.constraints} + ) + + if include_nil_filter_field do + {AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^field, ^expr and not is_nil(^include_nil_filter_field)))} + else + {AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^field, ^expr))} + end + else + if include_nil_filter_field do + {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} + else + {query, field} + end + end + end + + defp select_or_merge(query, aggregate_name, casted) do + query = + if query.select do + query + else + Ecto.Query.select(query, %{}) + end + + Ecto.Query.select_merge(query, ^%{aggregate_name => casted}) + end + + def aggregate_field_ref(aggregate, resource, relationship_path, query, first_relationship) do + if aggregate.kind == :count && !aggregate.field do + nil + else + %Ash.Query.Ref{ + attribute: aggregate_field(aggregate, resource, query), + relationship_path: relationship_path, + resource: query.__ash_bindings__.resource + } + |> case do + %{attribute: %Ash.Resource.Aggregate{}} = ref when not is_nil(first_relationship) -> + if first_relationship do + %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} + else + ref + end + + %{attribute: %Ash.Query.Aggregate{}} = ref when not is_nil(first_relationship) -> + if first_relationship do + %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} + else + ref + end + + other -> + other + end + end + end + + defp read_action_matches_relationship_default?(_resource, %{relationship_path: []}), do: true + + defp read_action_matches_relationship_default?(resource, aggregate) do + last_relationship = + resource + |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) + |> List.last() + + default_action_name = + last_relationship.read_action || + Ash.Resource.Info.primary_action!(last_relationship.destination, :read).name + + case aggregate.query && aggregate.query.action do + nil -> true + action -> action.name == default_action_name + end + end + + defp override_read_action(relationship, action_name) do + if relationship.read_action == action_name do + relationship + else + relationship + |> Map.put(:read_action, action_name) + |> Map.replace(:read_action_arguments, %{}) + end + end + + defp single_path?(_, []), do: true + + defp single_path?(resource, [relationship | rest]) do + relationship = Ash.Resource.Info.relationship(resource, relationship) + + !Map.get(relationship, :from_many?) && + (relationship.type == :belongs_to || + has_one_with_identity?(relationship)) && + single_path?(relationship.destination, rest) + end + + defp has_one_with_identity?(%{type: :has_one, from_many?: false} = relationship) do + Ash.Resource.Info.primary_key(relationship.destination) == [ + relationship.destination_attribute + ] || + relationship.destination + |> Ash.Resource.Info.identities() + |> Enum.any?(fn %{keys: keys} -> + keys == [relationship.destination_attribute] + end) + end + + defp has_one_with_identity?(_), do: false + + @doc false + def aggregate_field(aggregate, resource, query) do + if is_atom(aggregate.field) do + case Ash.Resource.Info.field( + resource, + aggregate.field || List.first(Ash.Resource.Info.primary_key(resource)) + ) do + %Ash.Resource.Calculation{calculation: {module, opts}} = calculation -> + calc_type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + calculation.type, + Map.get(calculation, :constraints, []), + :calculation + ) + + AshSql.Expr.validate_type!(query, calc_type, "#{inspect(calculation.name)}") + + {:ok, query_calc} = + Ash.Query.Calculation.new( + calculation.name, + module, + opts, + calculation.type, + calculation.constraints + ) + + Ash.Actions.Read.add_calc_context( + query_calc, + aggregate.context.actor, + aggregate.context.authorize?, + aggregate.context.tenant, + aggregate.context.tracer, + query.__ash_bindings__[:domain], + aggregate.query.resource, + parent_stack: [ + query.__ash_bindings__.resource | query.__ash_bindings__[:parent_resources] || [] + ] + ) + + nil -> + raise "no such aggregate field: #{inspect(resource)}.#{aggregate.field}" + + other -> + other + end + else + aggregate.field + end + end + + def wrap_in_subquery_for_aggregates(query) do + resource = query.__ash_bindings__.resource + selected_by_default = Ash.Resource.Info.selected_by_default_attribute_names(resource) + + selected_fields = + query.__ash_bindings__[:select] || + extract_selected_fields(query, resource, selected_by_default) + + all_attr_names = + resource + |> Ash.Resource.Info.attribute_names() + |> MapSet.to_list() + + # An upgraded combination query's source only exposes the default + # attributes plus the combination fieldset; others can't be re-selected. + available_attr_names = + if query.__ash_bindings__[:subquery_upgrade?] do + MapSet.union( + selected_by_default, + MapSet.new(query.__ash_bindings__[:already_selected] || []) + ) + else + MapSet.new(all_attr_names) + end + + to_select = + Enum.reject(all_attr_names, &(&1 in selected_fields or &1 not in available_attr_names)) + + query_with_all_attrs = + case query.select do + %Ecto.Query.SelectExpr{expr: {:merge, _, [l, {:%{}, [], kw}]}} -> + put_in( + query.select.expr, + {:merge, [], + [ + l, + {:%{}, [], + kw ++ + Enum.map( + to_select, + &{&1, + {{:., [], [{:&, [], [query.__ash_bindings__.root_binding]}, &1]}, [], []}} + )} + ]} + ) + + _ -> + from(row in query, + select_merge: struct(row, ^to_select) + ) + end + + # Flatten nested calculations/aggregates maps before creating subquery + # Ecto doesn't allow nested maps in subquery select expressions + {calculations_require_rewrite, aggregates_require_rewrite, query_with_all_attrs} = + AshSql.Query.rewrite_nested_selects(query_with_all_attrs) + + # After flattening, we need to: + # 1. Use the updated select_calculations from the rewritten query (which excludes :calculations) + # 2. Add the flattened calculation/aggregate field names to the select + flattened_calc_fields = Map.keys(calculations_require_rewrite) + flattened_agg_fields = Map.keys(aggregates_require_rewrite) + + # Get select_calculations from the rewritten query (it has :calculations removed) + select_calculations = + (query_with_all_attrs.__ash_bindings__[:select_calculations] || []) -- [:calculations] + + select_aggregates = + (query_with_all_attrs.__ash_bindings__[:select_aggregates] || []) -- [:aggregates] + + reselected_fields = + Enum.concat([ + selected_fields, + select_calculations, + select_aggregates, + flattened_calc_fields, + flattened_agg_fields + ]) + + # Upgraded combination queries return maps, which `struct/2` rejects, and + # `map/2` would drop field type info — merge each field explicitly instead. + subquery_query = + if query.__ash_bindings__[:subquery_upgrade?] do + root_binding = query.__ash_bindings__.root_binding + + base = + from(row in subquery(query_with_all_attrs), + as: ^root_binding, + select: %{} + ) + + Enum.reduce(reselected_fields, base, fn field, q -> + from(row in q, select_merge: %{^field => field(as(^root_binding), ^field)}) + end) + else + from(row in subquery(query_with_all_attrs), + as: ^query.__ash_bindings__.root_binding, + select: struct(row, ^reselected_fields) + ) + end + + root_binding = query.__ash_bindings__.root_binding + + only_root_binding = %{ + root_binding => query.__ash_bindings__.bindings[root_binding] + } + + new_bindings = + query.__ash_bindings__ + |> Map.put(:bindings, only_root_binding) + |> Map.delete(:__order__?) + |> Map.update( + :calculations_require_rewrite, + calculations_require_rewrite, + &Map.merge(&1, calculations_require_rewrite) + ) + |> Map.update( + :aggregates_require_rewrite, + aggregates_require_rewrite, + &Map.merge(&1, aggregates_require_rewrite) + ) + + Map.put(subquery_query, :__ash_bindings__, new_bindings) + end + + defp extract_selected_fields( + %{select: %Ecto.Query.SelectExpr{expr: expr, take: take}}, + resource, + all_attribute_names + ) do + Enum.uniq(extract_fields_from_expr(expr, resource, take, all_attribute_names)) + end + + defp extract_fields_from_expr(expr, resource, take, all_attribute_names) do + case expr do + {:&, [], [ix]} -> + case take do + %{^ix => {:struct, fields}} when is_list(fields) -> + fields + + %{^ix => {:map, fields}} when is_list(fields) -> + fields + + take when take == %{} -> + all_attribute_names + + _ -> + [] + end + + {:%{}, [], fields} -> + Enum.map(fields, fn {field_name, _} -> field_name end) + + {:%, [], [_struct, {:%{}, [], fields}]} -> + Enum.map(fields, fn {field_name, _} -> field_name end) + + {:merge, _, [sel1, sel2]} -> + Enum.concat( + extract_fields_from_expr(sel1, resource, take, all_attribute_names), + extract_fields_from_expr(sel2, resource, take, all_attribute_names) + ) + + _other -> + all_attribute_names + end + end +end From 98089d212ce58f02a9979ded93a82da0725df766 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:08:38 -0700 Subject: [PATCH 02/14] refactor(aggregates): extract lateral aggregate queries Move root aggregate-query execution into `AshSql.Aggregate.Lateral.Query` and keep `AshSql.AggregateQuery` as the public entry point. Implementation notes: - preserve the existing aggregate-query API; - normalize bindings once in the facade; - pass the prepared query to the lateral implementation; - remove the duplicate binding initialization from the extracted module. This is a structural extraction with no intended query behavior change. --- lib/aggregate/lateral/query.ex | 260 +++++++++++++++++++++++++++++++++ lib/aggregate_query.ex | 260 ++------------------------------- 2 files changed, 273 insertions(+), 247 deletions(-) create mode 100644 lib/aggregate/lateral/query.ex diff --git a/lib/aggregate/lateral/query.ex b/lib/aggregate/lateral/query.ex new file mode 100644 index 0000000..0aa13f0 --- /dev/null +++ b/lib/aggregate/lateral/query.ex @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Lateral.Query do + @moduledoc false + import Ecto.Query, only: [from: 2, subquery: 1] + + def run_aggregate_query(original_query, aggregates, resource, implementation) do + {can_group, cant_group} = + aggregates + |> Enum.split_with(&AshSql.Aggregate.can_group?(resource, &1, original_query)) + |> case do + {[one], cant_group} -> {[], [one | cant_group]} + {can_group, cant_group} -> {can_group, cant_group} + end + + {global_filter, can_group} = + AshSql.Aggregate.extract_shared_filters(can_group) + + query = + case global_filter do + {:ok, global_filter} -> + AshSql.Filter.filter(original_query, global_filter, resource) + + :error -> + {:ok, original_query} + end + + case query do + {:error, error} -> + {:error, error} + + {:ok, query} -> + query = + if query.distinct || query.limit do + query = + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(query), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + |> Map.put(:__ash_bindings__, query.__ash_bindings__) + + group_query = + Enum.reduce( + can_group, + query, + fn agg, query -> + first_relationship = + Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) + + AshSql.Aggregate.add_subquery_aggregate_select( + query, + agg.relationship_path |> Enum.drop(1), + agg, + resource, + false, + first_relationship + ) + end + ) + + result = + case can_group do + [] -> + %{} + + _ -> + repo = AshSql.dynamic_repo(resource, implementation, query) + repo.one(group_query, AshSql.repo_opts(repo, implementation, nil, nil, resource)) + end + + {:ok, add_single_aggs(result, resource, query, cant_group, implementation)} + end + end + + def add_single_aggs(result, resource, query, cant_group, implementation) do + Enum.reduce(cant_group, result, fn + %{kind: :exists} = agg, result -> + {:ok, filtered} = + case agg do + %{query: %{filter: filter}} when not is_nil(filter) -> + AshSql.Filter.filter(query, filter, resource) + + _ -> + {:ok, query} + end + + filtered = + if filtered.distinct || filtered.limit do + filtered = + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + + repo = AshSql.dynamic_repo(resource, implementation, filtered) + + Map.put( + result || %{}, + agg.name, + repo.exists?(filtered, AshSql.repo_opts(repo, implementation, nil, nil, resource)) + ) + + agg, result -> + {:ok, filtered} = + case agg do + %{query: %{filter: filter}} when not is_nil(filter) -> + AshSql.Filter.filter(query, filter, resource) + + _ -> + {:ok, query} + end + + filtered = + if filtered.distinct do + in_query = filtered |> Ecto.Query.exclude(:distinct) |> Ecto.Query.exclude(:select) + + dynamic = + Enum.reduce(Ash.Resource.Info.primary_key(resource), nil, fn key, dynamic -> + if dynamic do + Ecto.Query.dynamic( + [row], + ^dynamic and + field(parent_as(^query.__ash_bindings__.root_binding), ^key) == + field(row, ^key) + ) + else + Ecto.Query.dynamic( + [row], + field(parent_as(^query.__ash_bindings__.root_binding), ^key) == + field(row, ^key) + ) + end + end) + + in_query = + from(row in in_query, where: ^dynamic) + + in_query = Ecto.Query.exclude(in_query, :distinct) + + # `query.from.source` is only `{table, schema}`, so rebuilding from it + # drops the tenant schema prefix (`strategy(:context)` multitenancy). + # Restore it so the outer query reads the same schema as the original. + from(row in query.from.source, + as: ^query.__ash_bindings__.root_binding, + where: exists(in_query) + ) + |> Map.put(:prefix, query.prefix) + |> Map.update!(:from, &Map.put(&1, :prefix, query.from.prefix)) + else + filtered + end + + filtered = + if filtered.limit do + filtered = + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + + first_relationship = + Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) + + filtered = AshSql.Bindings.default_bindings(filtered, resource, implementation) + + ref = + AshSql.Aggregate.aggregate_field_ref( + agg, + Ash.Resource.Info.related(resource, agg.relationship_path), + agg.relationship_path, + filtered, + first_relationship + ) + + {:ok, filtered} = + if ref do + {:ok, filtered} = + case ref.attribute do + %struct{} = agg when struct in [Ash.Query.Aggregate, Ash.Resource.Aggregate] -> + AshSql.Aggregate.add_aggregates( + filtered, + [agg], + resource, + false, + filtered.__ash_bindings__.root_binding + ) + + %Ash.Query.Calculation{} = calc -> + used_aggregates = Ash.Filter.used_aggregates(calc, []) + + with {:ok, filtered} <- AshSql.Join.join_all_relationships(filtered, calc, []) do + AshSql.Aggregate.add_aggregates( + filtered, + used_aggregates, + resource, + false, + filtered.__ash_bindings__.root_binding + ) + end + + _other -> + {:ok, filtered} + end + + AshSql.Join.join_all_relationships(filtered, ref) + else + {:ok, filtered} + end + + query = + AshSql.Aggregate.add_subquery_aggregate_select( + filtered, + agg.relationship_path |> Enum.drop(1), + %{agg | query: %{agg.query | filter: nil}}, + resource, + true, + first_relationship + ) + + repo = AshSql.dynamic_repo(resource, implementation, query) + + Map.merge( + result || %{}, + repo.one( + query, + AshSql.repo_opts(repo, query.__ash_bindings__.sql_behaviour, nil, nil, resource) + ) + ) + end) + end +end diff --git a/lib/aggregate_query.ex b/lib/aggregate_query.ex index 41594bd..d1ccb04 100644 --- a/lib/aggregate_query.ex +++ b/lib/aggregate_query.ex @@ -4,260 +4,26 @@ defmodule AshSql.AggregateQuery do @moduledoc false - import Ecto.Query, only: [from: 2, subquery: 1] def run_aggregate_query(original_query, aggregates, resource, implementation) do original_query = AshSql.Bindings.default_bindings(original_query, resource, implementation) - {can_group, cant_group} = - aggregates - |> Enum.split_with(&AshSql.Aggregate.can_group?(resource, &1, original_query)) - |> case do - {[one], cant_group} -> {[], [one | cant_group]} - {can_group, cant_group} -> {can_group, cant_group} - end - - {global_filter, can_group} = - AshSql.Aggregate.extract_shared_filters(can_group) - - query = - case global_filter do - {:ok, global_filter} -> - AshSql.Filter.filter(original_query, global_filter, resource) - - :error -> - {:ok, original_query} - end - - case query do - {:error, error} -> - {:error, error} - - {:ok, query} -> - query = - if query.distinct || query.limit do - query = - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(query), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - |> Map.put(:__ash_bindings__, query.__ash_bindings__) - - group_query = - Enum.reduce( - can_group, - query, - fn agg, query -> - first_relationship = - Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) - - AshSql.Aggregate.add_subquery_aggregate_select( - query, - agg.relationship_path |> Enum.drop(1), - agg, - resource, - false, - first_relationship - ) - end - ) - - result = - case can_group do - [] -> - %{} - - _ -> - repo = AshSql.dynamic_repo(resource, implementation, query) - repo.one(group_query, AshSql.repo_opts(repo, implementation, nil, nil, resource)) - end - - {:ok, add_single_aggs(result, resource, query, cant_group, implementation)} - end + AshSql.Aggregate.Lateral.Query.run_aggregate_query( + original_query, + aggregates, + resource, + implementation + ) end def add_single_aggs(result, resource, query, cant_group, implementation) do - Enum.reduce(cant_group, result, fn - %{kind: :exists} = agg, result -> - {:ok, filtered} = - case agg do - %{query: %{filter: filter}} when not is_nil(filter) -> - AshSql.Filter.filter(query, filter, resource) - - _ -> - {:ok, query} - end - - filtered = - if filtered.distinct || filtered.limit do - filtered = - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - - repo = AshSql.dynamic_repo(resource, implementation, filtered) - - Map.put( - result || %{}, - agg.name, - repo.exists?(filtered, AshSql.repo_opts(repo, implementation, nil, nil, resource)) - ) - - agg, result -> - {:ok, filtered} = - case agg do - %{query: %{filter: filter}} when not is_nil(filter) -> - AshSql.Filter.filter(query, filter, resource) - - _ -> - {:ok, query} - end - - filtered = - if filtered.distinct do - in_query = filtered |> Ecto.Query.exclude(:distinct) |> Ecto.Query.exclude(:select) - - dynamic = - Enum.reduce(Ash.Resource.Info.primary_key(resource), nil, fn key, dynamic -> - if dynamic do - Ecto.Query.dynamic( - [row], - ^dynamic and - field(parent_as(^query.__ash_bindings__.root_binding), ^key) == - field(row, ^key) - ) - else - Ecto.Query.dynamic( - [row], - field(parent_as(^query.__ash_bindings__.root_binding), ^key) == - field(row, ^key) - ) - end - end) - - in_query = - from(row in in_query, where: ^dynamic) - - in_query = Ecto.Query.exclude(in_query, :distinct) - - # `query.from.source` is only `{table, schema}`, so rebuilding from it - # drops the tenant schema prefix (`strategy(:context)` multitenancy). - # Restore it so the outer query reads the same schema as the original. - from(row in query.from.source, - as: ^query.__ash_bindings__.root_binding, - where: exists(in_query) - ) - |> Map.put(:prefix, query.prefix) - |> Map.update!(:from, &Map.put(&1, :prefix, query.from.prefix)) - else - filtered - end - - filtered = - if filtered.limit do - filtered = - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - - first_relationship = - Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) - - filtered = AshSql.Bindings.default_bindings(filtered, resource, implementation) - - ref = - AshSql.Aggregate.aggregate_field_ref( - agg, - Ash.Resource.Info.related(resource, agg.relationship_path), - agg.relationship_path, - filtered, - first_relationship - ) - - {:ok, filtered} = - if ref do - {:ok, filtered} = - case ref.attribute do - %struct{} = agg when struct in [Ash.Query.Aggregate, Ash.Resource.Aggregate] -> - AshSql.Aggregate.add_aggregates( - filtered, - [agg], - resource, - false, - filtered.__ash_bindings__.root_binding - ) - - %Ash.Query.Calculation{} = calc -> - used_aggregates = Ash.Filter.used_aggregates(calc, []) - - with {:ok, filtered} <- AshSql.Join.join_all_relationships(filtered, calc, []) do - AshSql.Aggregate.add_aggregates( - filtered, - used_aggregates, - resource, - false, - filtered.__ash_bindings__.root_binding - ) - end - - _other -> - {:ok, filtered} - end - - AshSql.Join.join_all_relationships(filtered, ref) - else - {:ok, filtered} - end - - query = - AshSql.Aggregate.add_subquery_aggregate_select( - filtered, - agg.relationship_path |> Enum.drop(1), - %{agg | query: %{agg.query | filter: nil}}, - resource, - true, - first_relationship - ) - - repo = AshSql.dynamic_repo(resource, implementation, query) - - Map.merge( - result || %{}, - repo.one( - query, - AshSql.repo_opts(repo, query.__ash_bindings__.sql_behaviour, nil, nil, resource) - ) - ) - end) + AshSql.Aggregate.Lateral.Query.add_single_aggs( + result, + resource, + query, + cant_group, + implementation + ) end end From 70c6c3f243f4acff68af16446efc19b873e7ece9 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:08:47 -0700 Subject: [PATCH 03/14] feat(aggregates): define the strategy adapter contract Introduce the adapter callbacks that select an aggregate planner and provide dialect-specific grouped list expressions. Implementation notes: - `aggregate_strategy/1` defaults to `:lateral`; - adapters can opt into the new `:grouped` strategy; - `grouped_list_aggregate/2` keeps list SQL and result representation in the adapter; - both callbacks remain overridable through `AshSql.Implementation`. Existing adapters continue to use lateral aggregates without any required changes. --- lib/implementation.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/implementation.ex b/lib/implementation.ex index 090ca03..d15e9d0 100644 --- a/lib/implementation.ex +++ b/lib/implementation.ex @@ -42,6 +42,7 @@ defmodule AshSql.Implementation do {list(term), term} | list(term) @callback list_aggregate(Ash.Resource.t()) :: String.t() | nil + @callback grouped_list_aggregate(term, include_nil? :: boolean) :: term | nil @callback multicolumn_distinct?() :: boolean @@ -53,6 +54,7 @@ defmodule AshSql.Implementation do @callback strpos_function() :: String.t() @callback type_expr(expr :: term, type :: term) :: term @callback ref_cast_type(type :: term) :: term + @callback aggregate_strategy(Ash.Resource.t()) :: :lateral | :grouped @optional_callbacks determine_types: 3 @@ -67,6 +69,7 @@ defmodule AshSql.Implementation do def list_expr(_, _, _, _, _, _), do: :error def simple_join_first_aggregates(_), do: [] def list_aggregate(_), do: nil + def grouped_list_aggregate(_, _), do: nil def multicolumn_distinct?, do: true def require_ash_functions_for_or_and_and?, do: false def require_extension_for_citext, do: false @@ -74,6 +77,7 @@ defmodule AshSql.Implementation do def ilike?, do: true def equals_any?, do: true def storage_type(_, _), do: nil + def aggregate_strategy(_resource), do: :lateral # The cast type to use when casting a bare column reference, as opposed # to a value or a computed expression. Implementations can use this to @@ -109,10 +113,12 @@ defmodule AshSql.Implementation do require_ash_functions_for_or_and_and?: 0, require_extension_for_citext: 0, simple_join_first_aggregates: 1, + aggregate_strategy: 1, type_expr: 2, ref_cast_type: 1, storage_type: 2, list_aggregate: 1, + grouped_list_aggregate: 2, multicolumn_distinct?: 0 end end From 9fb0e616a4e9c1af6b28026c65dc33b40db2ca53 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:08:58 -0700 Subject: [PATCH 04/14] feat(aggregates): add grouped aggregate loading Add the grouped and windowed planner used by adapters that cannot load related aggregates through lateral joins. Implementation notes: - apply relationship limits and offsets per parent before aggregation; - resolve attributes, calculations, and aggregate fields through AshSQL expressions; - reuse the existing relationship and sort planners; - preserve aggregate type constraints; - delegate grouped list expressions to the selected adapter. The companion AshSQLite suite covers the supported aggregate kinds, relationship shapes, rich fields, defaults, and bounds. --- lib/aggregate.ex | 63 +- lib/aggregate/grouped.ex | 1722 ++++++++++++++++++++++++++++++++++++++ lib/sort.ex | 5 +- mix.exs | 1 + 4 files changed, 1789 insertions(+), 2 deletions(-) create mode 100644 lib/aggregate/grouped.ex diff --git a/lib/aggregate.ex b/lib/aggregate.ex index c59b5e2..fd1232b 100644 --- a/lib/aggregate.ex +++ b/lib/aggregate.ex @@ -18,7 +18,7 @@ defmodule AshSql.Aggregate do do: {:ok, query} def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do - AshSql.Aggregate.Lateral.add_aggregates( + strategy(query, resource).add_aggregates( query, aggregates, resource, @@ -76,6 +76,26 @@ defmodule AshSql.Aggregate do AshSql.Aggregate.Lateral.aggregate_field(aggregate, resource, query) end + def field_expression(query, aggregate, resource, relationship_path \\ []) do + ref = aggregate_field_ref(aggregate, resource, relationship_path, query, nil) + + with {:ok, query} <- add_field_dependencies(query, ref, resource) do + case ref do + nil -> + {:ok, query, nil} + + ref -> + case AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) do + {:error, error} -> + {:error, error} + + {expression, accumulator} -> + {:ok, AshSql.Bindings.merge_expr_accumulator(query, accumulator), expression} + end + end + end + end + def wrap_in_subquery_for_aggregates(query) do AshSql.Aggregate.Lateral.wrap_in_subquery_for_aggregates(query) end @@ -85,4 +105,45 @@ defmodule AshSql.Aggregate do defdelegate resource_aggregate_to_aggregate(resource, aggregate, opts \\ []), to: AshSql.Aggregate.Lateral + defp add_field_dependencies(query, nil, _resource), do: {:ok, query} + + defp add_field_dependencies(query, ref, resource) do + with {:ok, query} <- add_field_aggregates(query, ref.attribute, resource) do + AshSql.Join.join_all_relationships(query, ref) + end + end + + defp add_field_aggregates(query, %struct{} = aggregate, resource) + when struct in [Ash.Query.Aggregate, Ash.Resource.Aggregate] do + add_aggregates( + query, + [aggregate], + resource, + false, + query.__ash_bindings__.root_binding + ) + end + + defp add_field_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do + used_aggregates = Ash.Filter.used_aggregates(calculation, []) + + with {:ok, query} <- AshSql.Join.join_all_relationships(query, calculation, []) do + add_aggregates( + query, + used_aggregates, + resource, + false, + query.__ash_bindings__.root_binding + ) + end + end + + defp add_field_aggregates(query, _field, _resource), do: {:ok, query} + + defp strategy(query, resource) do + case query.__ash_bindings__.sql_behaviour.aggregate_strategy(resource) do + :lateral -> AshSql.Aggregate.Lateral + :grouped -> AshSql.Aggregate.Grouped + end + end end diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex new file mode 100644 index 0000000..58666e5 --- /dev/null +++ b/lib/aggregate/grouped.ex @@ -0,0 +1,1722 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Grouped do + @moduledoc false + + import Ecto.Query, only: [from: 2] + + @scalar_aggregate_kinds [:count, :sum, :avg, :max, :min, :exists] + @window_aggregate_kinds [:first, :list] + @supported_aggregate_kinds @scalar_aggregate_kinds ++ @window_aggregate_kinds ++ [:custom] + @window_value_field :__ash_sql_grouped_aggregate_value__ + @window_row_number_field :__ash_sql_grouped_aggregate_row_number__ + @window_count_field :__ash_sql_grouped_aggregate_count__ + @relationship_row_number_field :__ash_sql_grouped_relationship_row_number__ + @unrelated_join_field :__ash_sql_grouped_unrelated_join__ + + def add_aggregates(query, aggregates, resource, select?, _source_binding, _root_data) do + add_aggregates(query, aggregates, resource, select?: select?) + end + + def add_aggregates(query, aggregates, resource, opts \\ []) do + select? = Keyword.get(opts, :select?, true) + + do_add_aggregates(query, aggregates, resource, select?) + end + + def add_sort_aggregates(query, sort, _resource) when sort in [nil, []], do: {:ok, query} + + def add_sort_aggregates(query, sort, resource) do + with {:ok, aggregates} <- aggregates_from_sort(query, sort, resource) do + add_aggregates(query, aggregates, resource, select?: false) + end + end + + def relationship_filter_uses_parent?(%{filter: nil}), do: false + + def relationship_filter_uses_parent?(%{filter: filter}) do + filter_uses_parent?(filter) + end + + defp do_add_aggregates(query, [], _resource, _select?), do: {:ok, query} + + defp do_add_aggregates(query, aggregates, resource, select?) do + primary_key = Ash.Resource.Info.primary_key(resource) + + cond do + primary_key == [] -> + {:error, "AshSql cannot load aggregates on resources with no primary key"} + + Enum.any?(aggregates, &(not supported?(&1))) -> + {:error, + "AshSql only supports loading related count, sum, avg, min, max, exists, first, list and custom aggregates"} + + true -> + {already_added, remaining} = + aggregates + |> Enum.uniq_by(& &1.name) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__)) + + already_added_dynamics = + if select? do + Enum.map(already_added, &existing_aggregate_dynamic(&1, query.__ash_bindings__)) + else + [] + end + + remaining + |> Enum.group_by(&aggregate_group_key/1) + |> Enum.reduce_while({:ok, query, already_added_dynamics}, fn {relationship_path, + aggregates}, + {:ok, query, dynamics} -> + case add_aggregate_group( + query, + resource, + aggregate_relationship_path(relationship_path), + aggregates + ) do + {:ok, query, new_dynamics} -> + {:cont, {:ok, query, new_dynamics ++ dynamics}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + |> case do + {:ok, query, dynamics} -> + if select? do + {:ok, select_aggregates(query, dynamics)} + else + {:ok, query} + end + + {:error, error} -> + {:error, error} + end + end + end + + defp supported?(%{name: name}) when not is_atom(name), do: false + + defp supported?(%{kind: kind, related?: false}) when kind in @supported_aggregate_kinds do + true + end + + defp supported?(%{kind: kind, related?: true, relationship_path: path}) + when kind in @supported_aggregate_kinds do + match?([_ | _], path) + end + + defp supported?(_), do: false + + defp aggregate_group_key(aggregate) do + read_action = (aggregate.query.action && aggregate.query.action.name) || aggregate.read_action + + relationship_key = + case aggregate do + %{related?: false, query: %{resource: resource}} -> {:unrelated, resource} + %{relationship_path: relationship_path} -> {:related, relationship_path} + end + + {relationship_key, read_action, aggregate.join_filters || %{}, + aggregate_filter_group_key(aggregate), aggregate_kind_group_key(aggregate)} + end + + defp aggregate_relationship_path( + {{:related, relationship_path}, _read_action, _join_filters, _aggregate_filter_group, + _kind_group} + ) do + relationship_path + end + + defp aggregate_relationship_path( + {{:unrelated, _resource}, _read_action, _join_filters, _aggregate_filter_group, + _kind_group} + ) do + [] + end + + defp aggregate_kind_group_key(%{kind: kind, name: name}) when kind in @window_aggregate_kinds do + {kind, name} + end + + defp aggregate_kind_group_key(_aggregate), do: :shared + + defp aggregate_filter_group_key(aggregate) do + if aggregate_filter_uses_relationships?(aggregate) do + {:filter, aggregate.name} + else + :shared + end + end + + defp already_added?(aggregate, bindings) do + Enum.any?(bindings.bindings, fn + {_binding, %{type: :aggregate, aggregates: aggregates}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _binding -> + false + end) + end + + defp existing_aggregate_dynamic(aggregate, bindings) do + {binding, _aggregate_binding} = + Enum.find(bindings.bindings, fn + {_binding, %{type: :aggregate, aggregates: aggregates}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _binding -> + false + end) + + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, bindings.sql_behaviour)} + end + + defp aggregates_from_sort(query, sort, resource) do + sort + |> List.wrap() + |> Enum.reduce_while({:ok, []}, fn sort, {:ok, aggregates} -> + case sort_aggregates(query, sort, resource) do + {:ok, new_aggregates} -> + {:cont, {:ok, new_aggregates ++ aggregates}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + |> case do + {:ok, aggregates} -> {:ok, Enum.uniq(aggregates)} + {:error, error} -> {:error, error} + end + end + + defp sort_aggregates(query, {sort, _order}, resource) do + sort_key_aggregates(query, sort, resource) + end + + defp sort_aggregates(query, sort, resource) do + sort_key_aggregates(query, sort, resource) + end + + defp sort_key_aggregates(_query, %Ash.Query.Aggregate{} = aggregate, _resource) do + {:ok, [aggregate]} + end + + defp sort_key_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do + calculation_aggregates(query, calculation, resource) + end + + defp sort_key_aggregates(query, sort, resource) when is_atom(sort) do + case Ash.Resource.Info.field(resource, sort) do + %Ash.Resource.Aggregate{} = aggregate -> + query_aggregate(resource, aggregate) + + %Ash.Resource.Calculation{} = calculation -> + calculation_aggregates(query, calculation, resource) + + _ -> + {:ok, []} + end + end + + defp sort_key_aggregates(_query, _sort, _resource), do: {:ok, []} + + defp calculation_aggregates(query, %Ash.Resource.Calculation{} = calculation, resource) do + {module, opts} = calculation.calculation + + with {:ok, calculation} <- + Ash.Query.Calculation.new( + calculation.name, + module, + opts, + calculation.type, + calculation.constraints + ) do + calculation = + Ash.Actions.Read.add_calc_context( + calculation, + query.__ash_bindings__.context[:private][:actor], + query.__ash_bindings__.context[:private][:authorize?], + query.__ash_bindings__.context[:private][:tenant], + query.__ash_bindings__.context[:private][:tracer], + query.__ash_bindings__.context[:private][:domain], + query.__ash_bindings__.context[:private][:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + calculation_aggregates(query, calculation, resource) + end + end + + defp calculation_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do + calculation.opts + |> calculation.module.expression(calculation.context) + |> Ash.Filter.hydrate_refs(%{ + resource: resource, + aggregates: %{}, + parent_stack: query.__ash_bindings__[:parent_resources] || [], + calculations: %{}, + public?: false + }) + |> case do + {:ok, expression} -> + {:ok, Ash.Filter.used_aggregates(expression)} + + {:error, error} -> + {:error, error} + end + end + + defp query_aggregate(resource, aggregate) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + read_action = + aggregate.read_action || + Ash.Resource.Info.primary_action!(related, :read).name + + with %{valid?: true} = aggregate_query <- Ash.Query.for_read(related, read_action), + %{valid?: true} = aggregate_query <- + Ash.Query.build(aggregate_query, + filter: aggregate.filter, + sort: aggregate.sort + ), + {:ok, aggregate} <- + Ash.Query.Aggregate.new( + resource, + aggregate.name, + aggregate.kind, + path: aggregate.relationship_path, + query: aggregate_query, + field: aggregate.field, + default: aggregate.default, + filterable?: aggregate.filterable?, + type: aggregate.type, + sortable?: aggregate.sortable?, + include_nil?: aggregate.include_nil?, + constraints: aggregate.constraints, + implementation: aggregate.implementation, + uniq?: aggregate.uniq?, + read_action: read_action, + authorize?: aggregate.authorize?, + join_filters: aggregate.join_filters + ) do + {:ok, [aggregate]} + else + %{errors: errors} -> + {:error, errors} + + {:error, error} -> + {:error, error} + end + end + + defp add_aggregate_group(query, _resource, [], aggregates) do + if Enum.all?(aggregates, &(&1.related? == false)) do + do_add_unrelated_aggregate_group(query, aggregates) + else + {:error, "AshSql only supports loading unrelated aggregates with no relationship path"} + end + end + + defp add_aggregate_group(query, resource, relationship_path, aggregates) do + with {:ok, relationships} <- relationships(resource, relationship_path), + :ok <- validate_relationships(resource, relationship_path, relationships, aggregates) do + do_add_aggregate_group(query, relationships, aggregates) + end + end + + defp relationships(resource, relationship_path) do + {:ok, AshSql.Join.relationship_path_to_relationships(resource, relationship_path)} + end + + defp validate_relationships(resource, relationship_path, relationships, aggregates) do + cond do + Enum.any?(relationships, &match?(%{manual: {_, _}}, &1)) -> + {:error, "AshSql does not support loading aggregates over manual relationships"} + + Enum.any?(relationships, &Map.get(&1, :no_attributes?, false)) -> + {:error, "AshSql does not support loading aggregates over no_attributes? relationships"} + + Enum.any?(relationships, &relationship_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates over relationships with parent-dependent filters"} + + Enum.any?(relationships, &join_relationship_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates over many_to_many relationships with parent-dependent join filters"} + + Enum.any?(relationships, &limited_many_to_many?/1) -> + {:error, + "AshSql does not support loading aggregates over limited many_to_many relationships"} + + unsupported_multi_hop_many_to_many?(relationships, aggregates) -> + {:error, + "AshSql does not support loading aggregates over multi-hop paths that include many_to_many relationships"} + + Enum.empty?(relationships) -> + {:error, + "AshSql only supports loading aggregates over a relationship path from #{inspect(resource)}, got: #{inspect(relationship_path)}"} + + true -> + :ok + end + end + + defp limited_many_to_many?(%{type: :many_to_many} = relationship) do + limited_relationship?(relationship) + end + + defp limited_many_to_many?(_relationship), do: false + + defp unsupported_multi_hop_many_to_many?(relationships, aggregates) do + length(relationships) > 1 && + Enum.any?(relationships, &(&1.type == :many_to_many)) && + !supported_multi_hop_many_to_many?(relationships, aggregates) + end + + defp supported_multi_hop_many_to_many?(relationships, aggregates) do + List.last(relationships).type == :many_to_many && + Enum.count(relationships, &(&1.type == :many_to_many)) == 1 && + Enum.all?(aggregates, &(&1.kind in @scalar_aggregate_kinds)) + end + + defp do_add_unrelated_aggregate_group(query, aggregates) do + binding = query.__ash_bindings__.current + + with :ok <- validate_aggregate_filters(aggregates), + {:ok, aggregate_query} <- unrelated_aggregate_query(query, aggregates, binding) do + aggregate_query = Ecto.Query.subquery(aggregate_query) + + query = + from(_row in query, + left_join: aggregate in ^aggregate_query, + as: ^binding, + on: true + ) + + query = + AshSql.Bindings.add_binding(query, %{ + type: :aggregate, + path: [], + aggregates: aggregates + }) + + dynamics = + Enum.map(aggregates, fn aggregate -> + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, query.__ash_bindings__.sql_behaviour)} + end) + + {:ok, query, dynamics} + end + end + + defp do_add_aggregate_group(query, [first_relationship | _] = relationships, aggregates) do + binding = query.__ash_bindings__.current + + with :ok <- validate_aggregate_filters(aggregates), + {:ok, aggregate_query} <- + aggregate_query(query, relationships, aggregates, binding) do + aggregate_query = Ecto.Query.subquery(aggregate_query) + root_binding = query.__ash_bindings__.root_binding + + query = + from(_row in query, + left_join: aggregate in ^aggregate_query, + as: ^binding, + on: + field(as(^root_binding), ^first_relationship.source_attribute) == + field(aggregate, ^aggregate_join_attribute(first_relationship)) + ) + + query = + AshSql.Bindings.add_binding(query, %{ + type: :aggregate, + path: [], + aggregates: aggregates + }) + + dynamics = + Enum.map(aggregates, fn aggregate -> + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, query.__ash_bindings__.sql_behaviour)} + end) + + {:ok, query, dynamics} + end + end + + defp aggregate_query(parent_query, [relationship], [%{kind: kind} = aggregate], binding) + when kind in @window_aggregate_kinds do + case relationship do + %{type: :many_to_many} -> + many_to_many_window_aggregate_query(parent_query, relationship, aggregate, binding) + + relationship -> + related_window_aggregate_query(parent_query, relationship, aggregate, binding) + end + end + + defp aggregate_query( + parent_query, + [_ | _] = relationships, + [%{kind: kind} = aggregate], + binding + ) + when kind in @window_aggregate_kinds do + multi_hop_window_aggregate_query(parent_query, relationships, aggregate, binding) + end + + defp aggregate_query(parent_query, [relationship], aggregates, binding) do + case relationship do + %{type: :many_to_many} -> + many_to_many_aggregate_query(parent_query, relationship, aggregates, binding) + + relationship -> + related_aggregate_query(parent_query, relationship, aggregates, binding) + end + end + + defp aggregate_query(parent_query, relationships, aggregates, binding) do + case List.last(relationships) do + %{type: :many_to_many} -> + multi_hop_many_to_many_aggregate_query(parent_query, relationships, aggregates, binding) + + _relationship -> + multi_hop_aggregate_query(parent_query, relationships, aggregates, binding) + end + end + + defp unrelated_aggregate_query(parent_query, [%{kind: kind} = aggregate], binding) + when kind in @window_aggregate_kinds do + unrelated_window_aggregate_query(parent_query, aggregate, binding) + end + + defp unrelated_aggregate_query(parent_query, aggregates, binding) do + with {:ok, query} <- unrelated_query(parent_query, hd(aggregates), binding, filter?: false) do + root_binding = query.__ash_bindings__.root_binding + relationship = %{destination: hd(aggregates).query.resource} + + query = from(row in query, select: %{}) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp unrelated_window_aggregate_query(parent_query, aggregate, binding) do + with {:ok, query} <- unrelated_query(parent_query, aggregate, binding, filter?: true) do + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + @unrelated_join_field, + nil, + root_binding, + %{sort: []} + ) + end + end + + defp related_window_aggregate_query(parent_query, relationship, aggregate, binding) do + with {:ok, query} <- + related_window_query(parent_query, relationship, aggregate, binding, [ + relationship.name + ]) do + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + relationship.destination_attribute, + root_binding, + root_binding, + relationship + ) + end + end + + defp many_to_many_window_aggregate_query(parent_query, relationship, aggregate, binding) do + with {:ok, query} <- + related_window_query(parent_query, relationship, aggregate, binding, [ + relationship.name + ]) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- through_query(parent_query, relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^relationship.destination_attribute) + ) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: relationship + }) + + window_aggregate_query( + query, + aggregate, + relationship.source_attribute_on_join_resource, + through_binding, + root_binding, + relationship + ) + end + end + end + + defp multi_hop_window_aggregate_query(parent_query, relationships, aggregate, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_window_query( + parent_query, + final_relationship, + aggregate, + binding, + relationship_path + ), + {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, aggregate) do + first_relationship = hd(relationships) + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + first_relationship.destination_attribute, + first_related_binding, + root_binding, + final_relationship + ) + end + end + + defp related_aggregate_query(parent_query, relationship, aggregates, binding) do + with {:ok, query} <- + related_query(parent_query, relationship, hd(aggregates), binding, [relationship.name]) do + root_binding = query.__ash_bindings__.root_binding + + query = + from(row in query, + group_by: field(as(^root_binding), ^relationship.destination_attribute), + select: %{ + ^relationship.destination_attribute => + field(as(^root_binding), ^relationship.destination_attribute) + } + ) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp many_to_many_aggregate_query(parent_query, relationship, aggregates, binding) do + with {:ok, query} <- + related_query(parent_query, relationship, hd(aggregates), binding, [relationship.name]) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- through_query(parent_query, relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + ash_bindings = query.__ash_bindings__ + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^relationship.destination_attribute), + group_by: field(through, ^relationship.source_attribute_on_join_resource), + select: %{ + ^relationship.source_attribute_on_join_resource => + field(through, ^relationship.source_attribute_on_join_resource) + } + ) + |> Map.put(:__ash_bindings__, ash_bindings) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: relationship + }) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + end + + defp multi_hop_many_to_many_aggregate_query(parent_query, relationships, aggregates, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_query( + parent_query, + final_relationship, + hd(aggregates), + binding, + relationship_path + ) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- + through_query(parent_query, final_relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^final_relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^final_relationship.destination_attribute) + ) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: final_relationship + }) + + with {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, hd(aggregates), + current_binding: through_binding + ) do + first_relationship = hd(relationships) + + query = + from(row in query, + group_by: + field(as(^first_related_binding), ^first_relationship.destination_attribute), + select: %{ + ^first_relationship.destination_attribute => + field(as(^first_related_binding), ^first_relationship.destination_attribute) + } + ) + + root_binding = query.__ash_bindings__.root_binding + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, final_relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + end + end + + defp multi_hop_aggregate_query(parent_query, relationships, aggregates, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_query( + parent_query, + final_relationship, + hd(aggregates), + binding, + relationship_path + ), + {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, hd(aggregates)) do + first_relationship = hd(relationships) + + query = + from(row in query, + group_by: field(as(^first_related_binding), ^first_relationship.destination_attribute), + select: %{ + ^first_relationship.destination_attribute => + field(as(^first_related_binding), ^first_relationship.destination_attribute) + } + ) + + root_binding = query.__ash_bindings__.root_binding + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, final_relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp join_intermediate_relationships(parent_query, query, relationships, aggregate, opts \\ []) do + current_binding = Keyword.get(opts, :current_binding, query.__ash_bindings__.root_binding) + + relationships + |> Enum.zip(tl(relationships)) + |> Enum.with_index() + |> Enum.reverse() + |> Enum.reduce_while( + {:ok, query, current_binding, query.__ash_bindings__.current, nil}, + fn {{relationship, next_relationship}, index}, + {:ok, query, current_binding, next_binding, _first_related_binding} -> + path = + relationships + |> Enum.take(index + 1) + |> Enum.map(& &1.name) + + case intermediate_query(parent_query, relationship, next_binding, aggregate, path) do + {:ok, related_query} -> + related_query = Ecto.Query.subquery(related_query) + + on = intermediate_join_on(next_relationship, next_binding, current_binding) + + query = + from(row in query, + join: related in ^related_query, + as: ^next_binding, + on: ^on + ) + + {:cont, {:ok, query, next_binding, next_binding + 1, next_binding}} + + {:error, error} -> + {:halt, {:error, error}} + end + end + ) + |> case do + {:ok, query, _current_binding, _next_binding, first_related_binding} + when not is_nil(first_related_binding) -> + {:ok, query, first_related_binding} + + {:ok, _query, _current_binding, _next_binding, nil} -> + {:error, "AshSql could not build multi-hop aggregate joins"} + + {:error, error} -> + {:error, error} + end + end + + defp intermediate_join_on( + %{type: :many_to_many} = next_relationship, + related_binding, + current_binding + ) do + Ecto.Query.dynamic( + field(as(^related_binding), ^next_relationship.source_attribute) == + field(as(^current_binding), ^next_relationship.source_attribute_on_join_resource) + ) + end + + defp intermediate_join_on(next_relationship, related_binding, current_binding) do + Ecto.Query.dynamic( + field(as(^related_binding), ^next_relationship.source_attribute) == + field(as(^current_binding), ^next_relationship.destination_attribute) + ) + end + + defp related_query(parent_query, relationship, aggregate, binding, relationship_path) do + aggregate.query + |> Ash.Query.unset([:filter, :sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> limit_relationship_rows(relationship) + + {:error, error} -> + {:error, error} + end + end + + defp related_window_query(parent_query, relationship, aggregate, binding, relationship_path) do + aggregate.query + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> limit_relationship_rows(relationship) + + {:error, error} -> + {:error, error} + end + end + + defp unrelated_query(parent_query, aggregate, binding, opts) do + unset = + if Keyword.fetch!(opts, :filter?) do + [:sort, :distinct, :select, :limit, :offset] + else + [:filter, :sort, :distinct, :select, :limit, :offset] + end + + aggregate.query + |> Ash.Query.unset(unset) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp limit_relationship_rows(query, relationship) do + if limited_relationship?(relationship) do + root_binding = query.__ash_bindings__.root_binding + + with {:ok, order_by, query} <- + relationship_limit_order_by(query, relationship, root_binding) do + partition_by = + Ecto.Query.dynamic(field(as(^root_binding), ^relationship.destination_attribute)) + + ranked_query = + from(row in query, + windows: [ + ash_sql_grouped_relationship_window: [ + partition_by: ^partition_by, + order_by: ^order_by + ] + ], + select_merge: %{ + @relationship_row_number_field => + over(row_number(), :ash_sql_grouped_relationship_window) + } + ) + + filtered_query = + ranked_query + |> Ecto.Query.subquery() + |> relationship_row_filter(relationship, root_binding) + + {:ok, reset_relationship_bindings(filtered_query, query.__ash_bindings__)} + end + else + {:ok, query} + end + end + + defp limited_relationship?(relationship) do + is_integer(Map.get(relationship, :limit)) or + (Map.get(relationship, :offset) || 0) > 0 + end + + defp relationship_limit_order_by(query, relationship, root_binding) do + case List.wrap(Map.get(relationship, :sort)) do + [] -> + {:ok, [], query} + + sort -> + AshSql.Sort.sort( + query, + sort, + relationship.destination, + [], + root_binding, + :expressions + ) + end + end + + defp relationship_row_filter(query, relationship, root_binding) do + offset = Map.get(relationship, :offset) || 0 + row_number_field = @relationship_row_number_field + + case Map.get(relationship, :limit) do + limit when is_integer(limit) -> + from(row in query, + as: ^root_binding, + where: + field(row, ^row_number_field) > ^offset and + field(row, ^row_number_field) <= ^(offset + limit) + ) + + _limit -> + from(row in query, + as: ^root_binding, + where: field(row, ^row_number_field) > ^offset + ) + end + end + + defp reset_relationship_bindings(query, bindings) do + root_binding = bindings.root_binding + root = Map.fetch!(bindings.bindings, root_binding) + + Map.put(query, :__ash_bindings__, %{ + bindings + | bindings: %{root_binding => root}, + current: root_binding + 1, + expression_accumulator: %AshSql.Expr.ExprInfo{} + }) + end + + defp intermediate_query(parent_query, relationship, binding, aggregate, relationship_path) do + read_action = + relationship.read_action || + Ash.Resource.Info.primary_action!(relationship.destination, :read).name + + relationship.destination + |> Ash.Query.for_read(read_action) + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> limit_relationship_rows(relationship) + + {:error, error} -> + {:error, error} + end + end + + defp through_query(parent_query, relationship, binding) do + join_relationship = + Ash.Resource.Info.relationship(relationship.source, relationship.join_relationship) + + relationship.through + |> Ash.Query.new() + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.set_context(join_relationship.context) + |> Ash.Query.do_filter(join_relationship.filter) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp aggregate_join_attribute(%{type: :many_to_many} = relationship) do + relationship.source_attribute_on_join_resource + end + + defp aggregate_join_attribute(relationship), do: relationship.destination_attribute + + defp join_relationship_filter_uses_parent?(%{type: :many_to_many} = relationship) do + relationship.source + |> Ash.Resource.Info.relationship(relationship.join_relationship) + |> relationship_filter_uses_parent?() + end + + defp join_relationship_filter_uses_parent?(_relationship), do: false + + defp join_filter(%{join_filters: join_filters}, relationship_path) + when is_map(join_filters) do + Map.get(join_filters, relationship_path) + end + + defp join_filter(_aggregate, _relationship_path), do: nil + + defp validate_aggregate_filters(aggregates) do + cond do + Enum.any?(aggregates, &aggregate_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates with parent-dependent aggregate filters"} + + Enum.any?(aggregates, &aggregate_filter_uses_parent_dependent_relationship?/1) -> + {:error, + "AshSql does not support loading aggregates with filters that reference relationships with parent-dependent filters"} + + Enum.any?(aggregates, &aggregate_filter_uses_aggregates?/1) -> + {:error, + "AshSql does not support loading aggregates with aggregate filters that reference other aggregates"} + + Enum.any?(aggregates, &unsupported_to_many_aggregate_filter?/1) -> + {:error, + "AshSql does not support loading sum, avg, list, custom, or field-based count aggregates with filters that reference to-many relationships"} + + Enum.any?(aggregates, &join_filters_use_parent?/1) -> + {:error, "AshSql does not support loading aggregates with parent-dependent join filters"} + + true -> + :ok + end + end + + defp aggregate_filter_uses_parent?(%{query: %{filter: filter}}) do + filter_uses_parent?(filter) + end + + defp aggregate_filter_uses_parent_dependent_relationship?(%{ + query: %{filter: filter, resource: resource} + }) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?(&parent_dependent_relationship_path?(resource, &1)) + end + + defp aggregate_filter_uses_parent_dependent_relationship?(_aggregate), do: false + + defp aggregate_filter_uses_relationships?(%{query: %{filter: filter}}) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?() + end + + defp aggregate_filter_uses_relationships?(_aggregate), do: false + + defp aggregate_filter_uses_aggregates?(%{query: %{filter: filter}}) when not is_nil(filter) do + filter + |> Ash.Filter.used_aggregates([]) + |> Enum.any?() + end + + defp aggregate_filter_uses_aggregates?(_aggregate), do: false + + defp unsupported_to_many_aggregate_filter?(%{kind: :count, field: field} = aggregate) + when not is_nil(field) do + aggregate_filter_references_to_many_relationship?(aggregate) && !aggregate.uniq? + end + + defp unsupported_to_many_aggregate_filter?(%{kind: kind} = aggregate) + when kind in [:sum, :avg, :list, :custom] do + aggregate_filter_references_to_many_relationship?(aggregate) + end + + defp unsupported_to_many_aggregate_filter?(_aggregate), do: false + + defp aggregate_filter_references_to_many_relationship?(%{ + query: %{filter: filter, resource: resource} + }) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?(&to_many_relationship_path?(resource, &1)) + end + + defp aggregate_filter_references_to_many_relationship?(_aggregate), do: false + + defp aggregate_filter_relationship_paths(nil), do: [] + + defp aggregate_filter_relationship_paths(%{expression: nil}), do: [] + + defp aggregate_filter_relationship_paths(filter) do + Ash.Filter.relationship_paths(filter) + end + + defp parent_dependent_relationship_path?(_resource, []), do: false + + defp parent_dependent_relationship_path?(resource, [relationship_name | rest]) do + case Ash.Resource.Info.relationship(resource, relationship_name) do + nil -> + false + + relationship -> + relationship_filter_uses_parent?(relationship) || + parent_dependent_relationship_path?(relationship.destination, rest) + end + end + + defp to_many_relationship_path?(_resource, []), do: false + + defp to_many_relationship_path?(resource, [relationship_name | rest]) do + case Ash.Resource.Info.relationship(resource, relationship_name) do + %{cardinality: :many} -> + true + + nil -> + false + + relationship -> + to_many_relationship_path?(relationship.destination, rest) + end + end + + defp join_filters_use_parent?(%{join_filters: join_filters}) when is_map(join_filters) do + Enum.any?(join_filters, fn {_path, filter} -> filter_uses_parent?(filter) end) + end + + defp join_filters_use_parent?(_aggregate), do: false + + defp filter_uses_parent?(nil), do: false + + defp filter_uses_parent?(%{expression: nil}), do: false + + defp filter_uses_parent?(filter) do + Ash.Filter.find( + filter, + fn + %Ash.Query.Parent{} -> true + %Ash.Query.Call{name: :parent} -> true + _ -> false + end, + true, + true, + true + ) + |> case do + nil -> false + _ -> true + end + end + + defp window_aggregate_query( + query, + aggregate, + join_attribute, + partition_binding, + value_binding, + relationship + ) do + with :ok <- validate_window_aggregate(aggregate), + {:ok, sort} <- window_aggregate_sort(aggregate, relationship), + :ok <- validate_window_aggregate_sort(aggregate, sort), + {:ok, query, value} <- + AshSql.Aggregate.field_expression( + query, + aggregate, + Map.get(relationship, :destination, aggregate.query.resource) + ), + {:ok, order_by, query} <- + AshSql.Sort.sort( + query, + sort, + Map.get(relationship, :destination, aggregate.query.resource), + [], + value_binding, + :expressions + ) do + sql_behaviour = query.__ash_bindings__.sql_behaviour + + with {:ok, aggregate_value} <- window_aggregate_value(sql_behaviour, aggregate) do + query = + query + |> maybe_filter_window_nil_values(aggregate, value) + |> window_source_query(aggregate, join_attribute, partition_binding, value, order_by) + |> Ecto.Query.subquery() + |> window_result_query(aggregate, join_attribute, order_by, aggregate_value) + + {:ok, query} + end + end + end + + defp validate_window_aggregate(%{field: field, kind: kind}) + when kind in @window_aggregate_kinds and not is_nil(field) do + :ok + end + + defp validate_window_aggregate(%{name: name, field: field}) do + {:error, + "AshSql cannot load first or list aggregate #{inspect(name)} with field #{inspect(field)}"} + end + + defp validate_window_aggregate_sort(%{kind: :list, uniq?: true, field: field}, sort) do + if Enum.all?(sort, fn {sort_field, _order} -> sort_field == field end) do + :ok + else + {:error, + "AshSql only supports uniq list aggregates when sorting by the list aggregate field"} + end + end + + defp validate_window_aggregate_sort(_aggregate, _sort), do: :ok + + defp maybe_filter_window_nil_values(query, %{include_nil?: true}, _value), do: query + + defp maybe_filter_window_nil_values(query, _aggregate, value) do + filter = Ecto.Query.dynamic(not is_nil(^value)) + Ecto.Query.where(query, ^filter) + end + + defp window_source_query( + query, + aggregate, + join_attribute, + partition_binding, + value, + order_by + ) do + sort_selects = + order_by + |> Enum.with_index() + |> Map.new(fn {{_order, expression}, index} -> + {window_sort_field(index), expression} + end) + + select = + Map.merge( + %{ + join_attribute => window_join_field(partition_binding, join_attribute), + @window_value_field => value + }, + sort_selects + ) + + query = + if aggregate.kind == :list && aggregate.uniq? do + # This relies on validate_window_aggregate_sort/2 requiring uniq lists to + # sort by the listed field, so distinct applies to {parent, value}. + from(row in query, distinct: true) + else + query + end + + from(row in query, select: ^select) + end + + defp window_result_query(source_query, aggregate, join_attribute, sort, aggregate_value) do + order_by = + sort + |> Enum.with_index() + |> Enum.map(fn {{order, _expression}, index} -> + {ecto_sort_order(order), Ecto.Query.dynamic([row], field(row, ^window_sort_field(index)))} + end) + + partition_by = Ecto.Query.dynamic([row], field(row, ^join_attribute)) + + query = + from(row in source_query, + windows: [ + ash_sql_grouped_aggregate_window: [ + partition_by: ^partition_by, + order_by: ^order_by + ], + ash_sql_grouped_aggregate_partition_window: [ + partition_by: ^partition_by + ] + ], + select: %{ + ^join_attribute => field(row, ^join_attribute), + @window_row_number_field => over(row_number(), :ash_sql_grouped_aggregate_window), + @window_count_field => over(count(), :ash_sql_grouped_aggregate_partition_window) + } + ) + |> Ecto.Query.select_merge(^%{aggregate.name => aggregate_value}) + + row_filter = window_row_filter(aggregate) + + from(row in Ecto.Query.subquery(query), + where: ^row_filter, + select: %{ + ^join_attribute => field(row, ^join_attribute), + ^aggregate.name => field(row, ^aggregate.name) + } + ) + end + + defp window_row_filter(%{kind: :list}) do + row_number_field = @window_row_number_field + count_field = @window_count_field + + Ecto.Query.dynamic( + [row], + field(row, ^row_number_field) == field(row, ^count_field) + ) + end + + defp window_row_filter(_aggregate) do + row_number_field = @window_row_number_field + + Ecto.Query.dynamic([row], field(row, ^row_number_field) == 1) + end + + defp window_aggregate_value(sql_behaviour, %{kind: :first} = aggregate) do + value_field = @window_value_field + + value = + Ecto.Query.dynamic( + [row], + over(first_value(field(row, ^value_field)), :ash_sql_grouped_aggregate_window) + ) + + {:ok, maybe_type_dynamic(sql_behaviour, value, aggregate)} + end + + defp window_aggregate_value(sql_behaviour, %{kind: :list, include_nil?: true} = aggregate) do + value_field = @window_value_field + field = Ecto.Query.dynamic([row], field(row, ^value_field)) + + grouped_list_aggregate(sql_behaviour, field, aggregate) + end + + defp window_aggregate_value(sql_behaviour, %{kind: :list} = aggregate) do + value_field = @window_value_field + field = Ecto.Query.dynamic([row], field(row, ^value_field)) + + grouped_list_aggregate(sql_behaviour, field, aggregate) + end + + defp grouped_list_aggregate(sql_behaviour, field, aggregate) do + case sql_behaviour.grouped_list_aggregate(field, aggregate.include_nil?) do + nil -> + {:error, + "#{inspect(sql_behaviour)} does not implement list aggregates for the grouped aggregate strategy"} + + list_aggregate -> + {:ok, maybe_type_dynamic(sql_behaviour, list_aggregate, aggregate)} + end + end + + defp maybe_type_dynamic(_sql_behaviour, dynamic, %{type: nil}), do: dynamic + + defp maybe_type_dynamic(sql_behaviour, dynamic, aggregate) do + case aggregate_type(sql_behaviour, aggregate) do + nil -> dynamic + type -> sql_behaviour.type_expr(dynamic, type) + end + end + + defp aggregate_type(sql_behaviour, aggregate) do + AshSql.Expr.parameterized_type( + sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + end + + defp window_aggregate_sort(%{query: %{sort: sort}} = aggregate, relationship) do + sort = + cond do + sort not in [nil, []] -> + List.wrap(sort) + + relationship.sort not in [nil, []] -> + List.wrap(relationship.sort) + + true -> + [{aggregate.field, :asc}] + end + + sort + |> Enum.reduce_while({:ok, []}, fn + {field, order}, {:ok, acc} when is_atom(order) -> + {:cont, {:ok, [{field, order} | acc]}} + + field, {:ok, acc} when not is_nil(field) -> + {:cont, {:ok, [{field, :asc} | acc]}} + + sort, _acc -> + {:halt, + {:error, + "AshSql only supports first and list aggregate sorting by related fields, got: #{inspect(sort)}"}} + end) + |> case do + {:ok, sort} -> {:ok, Enum.reverse(sort)} + {:error, error} -> {:error, error} + end + end + + defp window_sort_field(index) do + :"__ash_sql_grouped_aggregate_sort_#{index}__" + end + + defp window_join_field(nil, _join_attribute) do + Ecto.Query.dynamic(fragment("1")) + end + + defp window_join_field(partition_binding, join_attribute) do + Ecto.Query.dynamic(field(as(^partition_binding), ^join_attribute)) + end + + defp ecto_sort_order(:asc), do: :asc + defp ecto_sort_order(:desc), do: :desc + defp ecto_sort_order(:asc_nils_first), do: :asc_nulls_first + defp ecto_sort_order(:asc_nils_last), do: :asc_nulls_last + defp ecto_sort_order(:desc_nils_first), do: :desc_nulls_first + defp ecto_sort_order(:desc_nils_last), do: :desc_nulls_last + defp ecto_sort_order(other), do: other + + defp aggregate_dynamic(query, relationship, %{kind: :exists} = aggregate, binding) do + with {:ok, query, count_dynamic} <- count_dynamic(query, relationship, aggregate, binding), + {:ok, query, count_dynamic} <- + maybe_filter_aggregate(query, aggregate, count_dynamic) do + {:ok, query, Ecto.Query.dynamic(^count_dynamic > 0)} + end + end + + defp aggregate_dynamic(query, relationship, %{kind: :count} = aggregate, binding) do + with {:ok, query, dynamic} <- count_dynamic(query, relationship, aggregate, binding), + {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(query, relationship, aggregate, _binding) + when aggregate.kind in [:sum, :avg, :max, :min] do + with {:ok, query, field} <- + AshSql.Aggregate.field_expression(query, aggregate, relationship.destination), + dynamic = scalar_aggregate_dynamic(aggregate.kind, field), + {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(query, _relationship, %{kind: :custom} = aggregate, binding) do + {module, opts} = aggregate.implementation + dynamic = module.dynamic(opts, binding) + + with {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(_query, _relationship, aggregate, _binding) do + {:error, + "AshSql cannot load aggregate #{inspect(aggregate.name)} with field #{inspect(aggregate.field)}"} + end + + defp scalar_aggregate_dynamic(:sum, field), do: Ecto.Query.dynamic(sum(^field)) + defp scalar_aggregate_dynamic(:avg, field), do: Ecto.Query.dynamic(avg(^field)) + defp scalar_aggregate_dynamic(:max, field), do: Ecto.Query.dynamic(max(^field)) + defp scalar_aggregate_dynamic(:min, field), do: Ecto.Query.dynamic(min(^field)) + + defp count_dynamic(query, relationship, %{field: nil} = aggregate, binding) do + if count_distinct?(aggregate) do + count_field = fieldless_count_field(relationship) + + {:ok, query, Ecto.Query.dynamic(count(field(as(^binding), ^count_field), :distinct))} + else + {:ok, query, Ecto.Query.dynamic(count())} + end + end + + defp count_dynamic(query, relationship, aggregate, _binding) do + with {:ok, query, field} <- + AshSql.Aggregate.field_expression(query, aggregate, relationship.destination) do + dynamic = + if count_distinct?(aggregate) do + Ecto.Query.dynamic(count(^field, :distinct)) + else + Ecto.Query.dynamic(count(^field)) + end + + {:ok, query, dynamic} + end + end + + defp fieldless_count_field(relationship) do + relationship.destination + |> Ash.Resource.Info.primary_key() + |> List.first() + |> case do + nil -> relationship.destination_attribute + field -> field + end + end + + defp count_distinct?(%{uniq?: true}), do: true + + defp count_distinct?(%{field: nil} = aggregate) do + aggregate_filter_references_to_many_relationship?(aggregate) + end + + defp count_distinct?(_aggregate), do: false + + defp maybe_filter_aggregate(query, aggregate, dynamic) do + case aggregate.query.filter do + nil -> + {:ok, query, dynamic} + + %{expression: nil} -> + {:ok, query, dynamic} + + filter -> + with {:ok, query} <- + AshSql.Join.join_all_relationships( + query, + filter, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) do + {filter_dynamic, acc} = + AshSql.Expr.dynamic_expr( + query, + filter, + Map.put(query.__ash_bindings__, :location, :aggregate), + false + ) + + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^dynamic, ^filter_dynamic))} + end + end + end + + defp maybe_default_aggregate(_query, dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_aggregate(_query, dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp loaded_aggregate_dynamic( + %{kind: :exists, default_value: nil} = aggregate, + binding, + _sql_behaviour + ) do + aggregate + |> loaded_aggregate_field(binding) + |> then(&Ecto.Query.dynamic(coalesce(^&1, false))) + end + + defp loaded_aggregate_dynamic(%{kind: :list} = aggregate, binding, sql_behaviour) do + type = aggregate_type(sql_behaviour, aggregate) + default_value = aggregate.default_value || [] + + aggregate + |> loaded_aggregate_field(binding) + |> then(fn field -> + if type do + default = list_default_expr(default_value, type, sql_behaviour) + + Ecto.Query.dynamic(coalesce(^field, ^default)) + |> sql_behaviour.type_expr(type) + else + Ecto.Query.dynamic(coalesce(^field, ^default_value)) + end + end) + end + + defp loaded_aggregate_dynamic(aggregate, binding, _sql_behaviour) do + aggregate + |> loaded_aggregate_field(binding) + |> maybe_default_loaded_aggregate(aggregate) + end + + defp maybe_default_loaded_aggregate(dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_loaded_aggregate(dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp list_default_expr(default_value, type, sql_behaviour) when is_list(default_value) do + default_value = Jason.encode!(default_value) + + Ecto.Query.dynamic(^default_value) + |> sql_behaviour.type_expr(type) + end + + defp list_default_expr(default_value, type, sql_behaviour) do + Ecto.Query.dynamic(^default_value) + |> sql_behaviour.type_expr(type) + end + + defp loaded_aggregate_field(aggregate, binding) do + Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + end + + defp select_aggregates(query, dynamics) do + {in_aggregates, in_body} = + Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) + + aggregates = + in_body + |> Map.new(fn {load, _name, dynamic} -> {load, dynamic} end) + + aggregates = + if Enum.empty?(in_aggregates) do + aggregates + else + Map.put( + aggregates, + :aggregates, + Map.new(in_aggregates, fn {_load, name, dynamic} -> {name, dynamic} end) + ) + end + + query = + if query.select do + query + else + from(row in query, select: %{}) + end + + Ecto.Query.select_merge(query, ^aggregates) + end +end diff --git a/lib/sort.ex b/lib/sort.ex index 2c1533d..5e333a5 100644 --- a/lib/sort.ex +++ b/lib/sort.ex @@ -305,7 +305,7 @@ defmodule AshSql.Sort do end) |> case do {:ok, [], query} -> - if type == :return do + if type in [:return, :expressions] do {:ok, [], query} else {:ok, query} @@ -313,6 +313,9 @@ defmodule AshSql.Sort do {:ok, sort_exprs, query} -> case type do + :expressions -> + {:ok, sort_exprs, query} + :return -> {:ok, order_to_fragments(sort_exprs), query} diff --git a/mix.exs b/mix.exs index eb005f0..6a29f0c 100644 --- a/mix.exs +++ b/mix.exs @@ -83,6 +83,7 @@ defmodule AshSql.MixProject do {:ash, ash_version("~> 3.33")}, {:ecto_sql, "~> 3.9"}, {:ecto, "~> 3.13 and >= 3.13.4"}, + {:jason, "~> 1.0"}, # dev/test dependencies {:igniter, "~> 0.5", only: [:dev, :test]}, {:simple_sat, "~> 0.1", only: [:dev, :test]}, From af74b669b2029cb6c646cd80482fd2f50cff0524 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:09:07 -0700 Subject: [PATCH 05/14] feat(aggregates): add grouped aggregate queries Route root aggregate queries through the selected strategy and add the grouped implementation for SQLite-style adapters. Implementation notes: - support count, sum, average, minimum, maximum, first, and exists; - apply filters, limits, offsets, and required ordering before aggregation; - preserve `first` nil handling, defaults, and Ash sort semantics; - resolve calculation and aggregate fields through shared expression handling; - derive fieldless distinct counts from single primary-key metadata and reject unsupported key shapes clearly. The companion AshSQLite regressions cover each of these query semantics end to end. --- lib/aggregate/grouped/query.ex | 226 +++++++++++++++++++++++++++++++++ lib/aggregate_query.ex | 9 +- 2 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 lib/aggregate/grouped/query.ex diff --git a/lib/aggregate/grouped/query.ex b/lib/aggregate/grouped/query.ex new file mode 100644 index 0000000..90fab94 --- /dev/null +++ b/lib/aggregate/grouped/query.ex @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Grouped.Query do + @moduledoc false + + import Ecto.Query, only: [from: 2, subquery: 1] + + @supported_kinds [:count, :first, :sum, :max, :min, :avg, :exists] + + def run_aggregate_query(original_query, aggregates, resource, implementation) do + aggregates + |> Enum.reduce_while({:ok, %{}}, fn aggregate, {:ok, acc} -> + case run_single_aggregate(original_query, aggregate, resource, implementation) do + {:ok, value} -> {:cont, {:ok, Map.put(acc, aggregate.name, value)}} + {:error, error} -> {:halt, {:error, error}} + end + end) + end + + defp run_single_aggregate(_original_query, %{kind: kind}, _resource, _implementation) + when kind not in @supported_kinds do + {:error, "AshSql grouped query aggregates do not support #{inspect(kind)} aggregates"} + end + + defp run_single_aggregate( + _original_query, + %{relationship_path: [_ | _]} = aggregate, + _resource, + _implementation + ) do + {:error, + "AshSql grouped query aggregates do not yet support relationship aggregate #{inspect(aggregate.name)}"} + end + + defp run_single_aggregate( + original_query, + %{kind: :exists} = aggregate, + resource, + implementation + ) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource) do + query = aggregate_base_query(query) + repo = AshSql.dynamic_repo(resource, implementation, query) + {:ok, repo.exists?(query, AshSql.repo_opts(repo, implementation, nil, nil, resource))} + end + end + + defp run_single_aggregate(original_query, %{kind: :first} = aggregate, resource, implementation) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource), + query = aggregate_base_query(query), + {:ok, query, field} <- AshSql.Aggregate.field_expression(query, aggregate, resource), + {:ok, query} <- sort_first(query, aggregate, resource) do + query = + query + |> Ecto.Query.exclude(:select) + |> Map.put(:windows, []) + |> maybe_filter_first_nil_values(aggregate, field) + |> Ecto.Query.limit(1) + |> Ecto.Query.select(^field) + + repo = AshSql.dynamic_repo(resource, implementation, query) + + value = repo.one(query, AshSql.repo_opts(repo, implementation, nil, nil, resource)) + + {:ok, maybe_default_value(value, aggregate)} + end + end + + defp run_single_aggregate(original_query, aggregate, resource, implementation) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource), + query = aggregate_base_query(query), + {:ok, query, dynamic} <- aggregate_dynamic(query, aggregate, resource) do + query = Ecto.Query.select(query, ^%{aggregate.name => dynamic}) + + repo = AshSql.dynamic_repo(resource, implementation, query) + + result = + query + |> repo.one(AshSql.repo_opts(repo, implementation, nil, nil, resource)) + |> Map.get(aggregate.name) + + {:ok, result} + end + end + + defp filtered_query(original_query, aggregate, resource) do + case aggregate.query.filter do + nil -> {:ok, original_query} + %{expression: nil} -> {:ok, original_query} + filter -> AshSql.Filter.filter(original_query, filter, resource) + end + end + + defp aggregate_base_query(query) do + if query.distinct || query.limit || query.offset do + query = + query + |> Ecto.Query.exclude(:select) + |> Map.put(:windows, []) + |> maybe_add_offset_limit() + |> maybe_exclude_subquery_order() + + from(row in subquery(query), as: ^query.__ash_bindings__.root_binding) + |> Map.put(:__ash_bindings__, query.__ash_bindings__) + else + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + end + end + + defp aggregate_dynamic(query, %{kind: :count, field: nil, uniq?: true} = aggregate, resource) do + case Ash.Resource.Info.primary_key(resource) do + [field] -> + dynamic = + Ecto.Query.dynamic( + count(field(as(^query.__ash_bindings__.root_binding), ^field), :distinct) + ) + + {:ok, query, dynamic} + + [] -> + {:error, + "AshSql grouped query aggregate #{inspect(aggregate.name)} requires a single primary key to count distinct records, but #{inspect(resource)} has no primary key"} + + fields -> + {:error, + "AshSql grouped query aggregate #{inspect(aggregate.name)} requires a single primary key to count distinct records, but #{inspect(resource)} has composite primary key #{inspect(fields)}"} + end + end + + defp aggregate_dynamic(query, %{kind: :count, field: nil}, _resource), + do: {:ok, query, Ecto.Query.dynamic(count())} + + defp aggregate_dynamic(query, %{kind: :count} = aggregate, resource) do + with {:ok, query, field} <- AshSql.Aggregate.field_expression(query, aggregate, resource) do + dynamic = + if aggregate.uniq? do + Ecto.Query.dynamic(count(^field, :distinct)) + else + Ecto.Query.dynamic(count(^field)) + end + + {:ok, query, dynamic} + end + end + + defp aggregate_dynamic(query, aggregate, resource) + when aggregate.kind in [:sum, :max, :min, :avg] do + with {:ok, query, field} <- AshSql.Aggregate.field_expression(query, aggregate, resource) do + dynamic = + case aggregate.kind do + :sum -> Ecto.Query.dynamic(sum(^field)) + :max -> Ecto.Query.dynamic(max(^field)) + :min -> Ecto.Query.dynamic(min(^field)) + :avg -> Ecto.Query.dynamic(avg(^field)) + end + + {:ok, query, + maybe_type_dynamic(query, maybe_default_dynamic(dynamic, aggregate), aggregate)} + end + end + + defp aggregate_dynamic(_query, aggregate, _resource) do + {:error, "AshSql grouped query aggregate #{inspect(aggregate.name)} is unsupported"} + end + + defp maybe_type_dynamic(query, dynamic, aggregate) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + if type do + query.__ash_bindings__.sql_behaviour.type_expr(dynamic, type) + else + dynamic + end + end + + defp maybe_default_dynamic(dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_dynamic(dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp sort_first(query, %{query: %{sort: sort}}, _resource) when sort in [nil, []], + do: {:ok, query} + + defp sort_first(query, %{query: %{sort: sort}}, resource) do + AshSql.Sort.sort( + query, + List.wrap(sort), + resource, + [], + query.__ash_bindings__.root_binding, + :direct + ) + end + + defp maybe_filter_first_nil_values(query, %{include_nil?: true}, _field), do: query + + defp maybe_filter_first_nil_values(query, _aggregate, field) do + filter = Ecto.Query.dynamic(not is_nil(^field)) + Ecto.Query.where(query, ^filter) + end + + defp maybe_default_value(nil, %{default_value: default_value}), do: default_value + defp maybe_default_value(value, _aggregate), do: value + + defp maybe_add_offset_limit(%{limit: nil, offset: offset} = query) when not is_nil(offset), + do: Ecto.Query.limit(query, -1) + + defp maybe_add_offset_limit(query), do: query + + defp maybe_exclude_subquery_order(%{limit: nil, offset: nil} = query), + do: Ecto.Query.exclude(query, :order_by) + + defp maybe_exclude_subquery_order(query), do: query +end diff --git a/lib/aggregate_query.ex b/lib/aggregate_query.ex index d1ccb04..4f51c05 100644 --- a/lib/aggregate_query.ex +++ b/lib/aggregate_query.ex @@ -9,7 +9,7 @@ defmodule AshSql.AggregateQuery do original_query = AshSql.Bindings.default_bindings(original_query, resource, implementation) - AshSql.Aggregate.Lateral.Query.run_aggregate_query( + strategy(original_query, resource).run_aggregate_query( original_query, aggregates, resource, @@ -26,4 +26,11 @@ defmodule AshSql.AggregateQuery do implementation ) end + + defp strategy(query, resource) do + case query.__ash_bindings__.sql_behaviour.aggregate_strategy(resource) do + :lateral -> AshSql.Aggregate.Lateral.Query + :grouped -> AshSql.Aggregate.Grouped.Query + end + end end From 06c708bba57ba850e66cb4bf5a1a541a05ccd6b6 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Tue, 21 Jul 2026 22:09:16 -0700 Subject: [PATCH 06/14] docs(aggregates): document aggregate strategies Explain how SQL adapters select between lateral and grouped aggregate planning. Documentation includes: - the default `:lateral` behavior; - the adapter opt-in for `:grouped`; - the adapter-owned grouped list callback; - the new strategy support in the changelog. The guide describes the final contract after both loaded and root aggregate paths are in place. --- CHANGELOG.md | 11 +++++++++++ README.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe5141..5e4b23c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ See [Conventional Commits](Https://conventionalcommits.org) for commit guideline +## Unreleased + +### Features: + +* add strategy-aware aggregate dispatch with lateral and grouped aggregate implementations + +### Improvements: + +* add grouped query aggregate support for root SQLite-compatible aggregate kinds +* declare `:jason` as a direct dependency for grouped list aggregate defaults + ## [v0.7.6](https://github.com/ash-project/ash_sql/compare/v0.7.5...v0.7.6) (2026-09-18) diff --git a/README.md b/README.md index a0d7e69..bcec39c 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,14 @@ def deps do ] end ``` + +## Aggregate Strategies + +`AshSql.Implementation` defaults aggregate planning to `:lateral`. SQL data +layers can override `aggregate_strategy/1` with `:grouped` when they need the +SQLite-style grouped aggregate implementation. + +The grouped strategy uses adapter-provided list aggregation. Implementations +that select `:grouped` must implement `grouped_list_aggregate/2` and return the +windowed list expression for their SQL dialect. AshSQLite uses SQLite's JSON +list representation for this callback. From 1f15af09d7f1cb4d5f36827a5f7968beec7580eb Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Fri, 18 Sep 2026 21:23:14 -0700 Subject: [PATCH 07/14] fix(aggregates): preserve attachment context and aggregate identity Normalize resource aggregates and SQL aliases before strategy dispatch. Scope alias reuse and expression lookup to the attachment path, and correlate grouped joins with the supplied source binding. Route nested lateral calls through the shared facade and cover alias reuse, path isolation, and actor/tenant propagation. --- README.md | 6 + lib/aggregate.ex | 23 +- lib/aggregate/common.ex | 187 ++++++++++ lib/aggregate/grouped.ex | 67 ++-- lib/aggregate/lateral.ex | 757 ++++++++++++++++----------------------- lib/expr.ex | 14 +- test/aggregate_test.exs | 55 +++ 7 files changed, 615 insertions(+), 494 deletions(-) create mode 100644 lib/aggregate/common.ex diff --git a/README.md b/README.md index bcec39c..0eae2ec 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,9 @@ The grouped strategy uses adapter-provided list aggregation. Implementations that select `:grouped` must implement `grouped_list_aggregate/2` and return the windowed list expression for their SQL dialect. AshSQLite uses SQLite's JSON list representation for this callback. + +The aggregate facade normalizes resource aggregates and SQL aliases before +dispatch. Both strategies preserve the source binding and attachment path, so +aggregates referenced through joined relationships attach to the related row. +Alias reuse is scoped to that path and follows the existing filter/sort identity +rules. Public aggregate names, including strings, are retained in results. diff --git a/lib/aggregate.ex b/lib/aggregate.ex index fd1232b..366c258 100644 --- a/lib/aggregate.ex +++ b/lib/aggregate.ex @@ -18,14 +18,17 @@ defmodule AshSql.Aggregate do do: {:ok, query} def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do - strategy(query, resource).add_aggregates( - query, - aggregates, - resource, - select?, - source_binding, - root_data - ) + with {:ok, query, aggregates} <- + AshSql.Aggregate.Common.normalize(query, aggregates, resource, root_data) do + strategy(query, resource).add_aggregates( + query, + aggregates, + resource, + select?, + source_binding, + root_data + ) + end end def extract_shared_filters(aggregates) do @@ -33,7 +36,7 @@ defmodule AshSql.Aggregate do end def next_aggregate_name(index) do - AshSql.Aggregate.Lateral.next_aggregate_name(index) + AshSql.Aggregate.Common.next_aggregate_name(index) end def can_group?(resource, aggregate, query) do @@ -103,7 +106,7 @@ defmodule AshSql.Aggregate do defdelegate limit_one_first_aggregate?(resource, aggregate), to: AshSql.Aggregate.Lateral defdelegate resource_aggregate_to_aggregate(resource, aggregate, opts \\ []), - to: AshSql.Aggregate.Lateral + to: AshSql.Aggregate.Common defp add_field_dependencies(query, nil, _resource), do: {:ok, query} diff --git a/lib/aggregate/common.ex b/lib/aggregate/common.ex new file mode 100644 index 0000000..87d431e --- /dev/null +++ b/lib/aggregate/common.ex @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Common do + @moduledoc false + + @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> + Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") + end) + + def normalize(query, aggregates, resource, root_data) do + path = attachment_path(root_data) + + with {:ok, aggregates} <- resource_aggregates_to_aggregates(resource, query, aggregates) do + {query, aggregates} = + Enum.reduce(aggregates, {query, []}, fn aggregate, {query, aggregates} -> + # SQL aliases belong to a definition at an attachment path. The + # public-name map alone only describes how to return loaded results. + key = {path, aggregate.resource, aggregate.name} + registry = Map.get(query.__ash_bindings__, :aggregate_registry, %{}) + definitions = Map.get(registry, key, []) + existing = Enum.find(definitions, &(not different_queries?(&1.query, aggregate.query))) + + {query, name} = + cond do + existing -> {query, existing.name} + is_atom(aggregate.name) && definitions == [] -> {query, aggregate.name} + true -> use_aggregate_name(query) + end + + query = + if name != aggregate.name && path == [] do + put_in(query.__ash_bindings__.aggregate_names[aggregate.name], name) + else + query + end + + aggregate = %{aggregate | name: name} + + registry = + if existing, do: registry, else: Map.put(registry, key, [aggregate | definitions]) + + query = + update_in(query.__ash_bindings__, fn bindings -> + bindings = Map.put(bindings, :aggregate_registry, registry) + + if path == [] do + Map.update!(bindings, :aggregate_defs, &Map.put(&1, name, aggregate)) + else + bindings + end + end) + + {query, [aggregate | aggregates]} + end) + + {:ok, query, aggregates} + end + end + + def attachment_path(nil), do: [] + def attachment_path({_, path}), do: path + + def name_for(aggregate, bindings, path) do + key = {List.wrap(bindings[:refs_at_path]) ++ path, aggregate.resource, aggregate.name} + + bindings + |> Map.get(:aggregate_registry, %{}) + |> Map.get(key, []) + |> Enum.find(&(not different_queries?(&1.query, aggregate.query))) + |> case do + nil -> bindings.aggregate_names[aggregate.name] || aggregate.name + existing -> existing.name + end + end + + defp use_aggregate_name(query) do + name = query.__ash_bindings__.current_aggregate_name + {put_in(query.__ash_bindings__.current_aggregate_name, next_aggregate_name(name)), name} + end + + defp different_queries?(nil, nil), do: false + defp different_queries?(nil, _), do: true + defp different_queries?(_, nil), do: true + + defp different_queries?(query1, query2) do + # Keep the upstream lateral identity rules when sharing normalization. + query1.filter != query2.filter || query1.sort != query2.sort + end + + defp resource_aggregates_to_aggregates(resource, query, aggregates) do + private_context = query.__ash_bindings__.context[:private] + + Enum.reduce_while(aggregates, {:ok, []}, fn + %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> + aggregate = + Ash.Actions.Read.add_calc_context( + aggregate, + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + aggregate, {:ok, aggregates} -> + resource + |> resource_aggregate_to_aggregate(aggregate, + actor: private_context[:actor], + tenant: private_context[:tenant] + ) + |> case do + {:ok, aggregate} -> + aggregate = + aggregate + |> Map.put(:load, aggregate.name) + |> Ash.Actions.Read.add_calc_context( + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + + @doc false + def resource_aggregate_to_aggregate(resource, aggregate, opts \\ []) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + read_action = + aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name + + with %{valid?: true} = aggregate_query <- + Ash.Query.for_read(related, read_action, %{}, + actor: opts[:actor], + tenant: opts[:tenant] + ), + %{valid?: true} = aggregate_query <- + Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do + Ash.Query.Aggregate.new( + resource, + aggregate.name, + aggregate.kind, + path: aggregate.relationship_path, + query: aggregate_query, + field: aggregate.field, + default: aggregate.default, + filterable?: aggregate.filterable?, + type: aggregate.type, + sortable?: aggregate.filterable?, + include_nil?: aggregate.include_nil?, + constraints: aggregate.constraints, + implementation: aggregate.implementation, + uniq?: aggregate.uniq?, + read_action: read_action, + authorize?: aggregate.authorize? + ) + else + %{errors: errors} -> + {:error, errors} + end + end + + def next_aggregate_name(i) do + @next_aggregate_names[i] || + raise Ash.Error.Framework.AssumptionFailed, + message: """ + All 1000 static names for aggregates have been used in a single query. + Congratulations, this means that you have gone so wildly beyond our imagination + of how much can fit into a single quer. Please file an issue and we will raise the limit. + """ + end +end diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex index 58666e5..0896338 100644 --- a/lib/aggregate/grouped.ex +++ b/lib/aggregate/grouped.ex @@ -16,14 +16,24 @@ defmodule AshSql.Aggregate.Grouped do @relationship_row_number_field :__ash_sql_grouped_relationship_row_number__ @unrelated_join_field :__ash_sql_grouped_unrelated_join__ - def add_aggregates(query, aggregates, resource, select?, _source_binding, _root_data) do - add_aggregates(query, aggregates, resource, select?: select?) + def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do + path = AshSql.Aggregate.Common.attachment_path(root_data) + + do_add_aggregates(query, aggregates, resource, select?, %{ + source_binding: source_binding, + path: path + }) end def add_aggregates(query, aggregates, resource, opts \\ []) do - select? = Keyword.get(opts, :select?, true) - - do_add_aggregates(query, aggregates, resource, select?) + AshSql.Aggregate.add_aggregates( + query, + aggregates, + resource, + Keyword.get(opts, :select?, true), + query.__ash_bindings__.root_binding, + nil + ) end def add_sort_aggregates(query, sort, _resource) when sort in [nil, []], do: {:ok, query} @@ -40,9 +50,9 @@ defmodule AshSql.Aggregate.Grouped do filter_uses_parent?(filter) end - defp do_add_aggregates(query, [], _resource, _select?), do: {:ok, query} + defp do_add_aggregates(query, [], _resource, _select?, _context), do: {:ok, query} - defp do_add_aggregates(query, aggregates, resource, select?) do + defp do_add_aggregates(query, aggregates, resource, select?, context) do primary_key = Ash.Resource.Info.primary_key(resource) cond do @@ -57,11 +67,14 @@ defmodule AshSql.Aggregate.Grouped do {already_added, remaining} = aggregates |> Enum.uniq_by(& &1.name) - |> Enum.split_with(&already_added?(&1, query.__ash_bindings__)) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, context.path)) already_added_dynamics = if select? do - Enum.map(already_added, &existing_aggregate_dynamic(&1, query.__ash_bindings__)) + Enum.map( + already_added, + &existing_aggregate_dynamic(&1, query.__ash_bindings__, context.path) + ) else [] end @@ -75,7 +88,8 @@ defmodule AshSql.Aggregate.Grouped do query, resource, aggregate_relationship_path(relationship_path), - aggregates + aggregates, + context ) do {:ok, query, new_dynamics} -> {:cont, {:ok, query, new_dynamics ++ dynamics}} @@ -152,9 +166,9 @@ defmodule AshSql.Aggregate.Grouped do end end - defp already_added?(aggregate, bindings) do + defp already_added?(aggregate, bindings, path) do Enum.any?(bindings.bindings, fn - {_binding, %{type: :aggregate, aggregates: aggregates}} -> + {_binding, %{type: :aggregate, aggregates: aggregates, path: ^path}} -> aggregate.name in Enum.map(aggregates, & &1.name) _binding -> @@ -162,10 +176,10 @@ defmodule AshSql.Aggregate.Grouped do end) end - defp existing_aggregate_dynamic(aggregate, bindings) do + defp existing_aggregate_dynamic(aggregate, bindings, path) do {binding, _aggregate_binding} = Enum.find(bindings.bindings, fn - {_binding, %{type: :aggregate, aggregates: aggregates}} -> + {_binding, %{type: :aggregate, aggregates: aggregates, path: ^path}} -> aggregate.name in Enum.map(aggregates, & &1.name) _binding -> @@ -314,18 +328,18 @@ defmodule AshSql.Aggregate.Grouped do end end - defp add_aggregate_group(query, _resource, [], aggregates) do + defp add_aggregate_group(query, _resource, [], aggregates, context) do if Enum.all?(aggregates, &(&1.related? == false)) do - do_add_unrelated_aggregate_group(query, aggregates) + do_add_unrelated_aggregate_group(query, aggregates, context) else {:error, "AshSql only supports loading unrelated aggregates with no relationship path"} end end - defp add_aggregate_group(query, resource, relationship_path, aggregates) do + defp add_aggregate_group(query, resource, relationship_path, aggregates, context) do with {:ok, relationships} <- relationships(resource, relationship_path), :ok <- validate_relationships(resource, relationship_path, relationships, aggregates) do - do_add_aggregate_group(query, relationships, aggregates) + do_add_aggregate_group(query, relationships, aggregates, context) end end @@ -384,7 +398,7 @@ defmodule AshSql.Aggregate.Grouped do Enum.all?(aggregates, &(&1.kind in @scalar_aggregate_kinds)) end - defp do_add_unrelated_aggregate_group(query, aggregates) do + defp do_add_unrelated_aggregate_group(query, aggregates, context) do binding = query.__ash_bindings__.current with :ok <- validate_aggregate_filters(aggregates), @@ -401,7 +415,7 @@ defmodule AshSql.Aggregate.Grouped do query = AshSql.Bindings.add_binding(query, %{ type: :aggregate, - path: [], + path: context.path, aggregates: aggregates }) @@ -415,28 +429,33 @@ defmodule AshSql.Aggregate.Grouped do end end - defp do_add_aggregate_group(query, [first_relationship | _] = relationships, aggregates) do + defp do_add_aggregate_group( + query, + [first_relationship | _] = relationships, + aggregates, + context + ) do binding = query.__ash_bindings__.current with :ok <- validate_aggregate_filters(aggregates), {:ok, aggregate_query} <- aggregate_query(query, relationships, aggregates, binding) do aggregate_query = Ecto.Query.subquery(aggregate_query) - root_binding = query.__ash_bindings__.root_binding + source_binding = context.source_binding query = from(_row in query, left_join: aggregate in ^aggregate_query, as: ^binding, on: - field(as(^root_binding), ^first_relationship.source_attribute) == + field(as(^source_binding), ^first_relationship.source_attribute) == field(aggregate, ^aggregate_join_attribute(first_relationship)) ) query = AshSql.Bindings.add_binding(query, %{ type: :aggregate, - path: [], + path: context.path, aggregates: aggregates }) diff --git a/lib/aggregate/lateral.ex b/lib/aggregate/lateral.ex index ac77450..cbb576c 100644 --- a/lib/aggregate/lateral.ex +++ b/lib/aggregate/lateral.ex @@ -8,10 +8,6 @@ defmodule AshSql.Aggregate.Lateral do require Ecto.Query import Ecto.Query, only: [from: 2, subquery: 1] - @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> - Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") - end) - def add_aggregates( query, aggregates, @@ -24,367 +20,324 @@ defmodule AshSql.Aggregate.Lateral do def add_aggregates(query, [], _, _, _, _), do: {:ok, query} def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do - case resource_aggregates_to_aggregates(resource, query, aggregates) do - {:ok, aggregates} -> - root_data_path = - case root_data do - {_, path} -> - path + root_data_path = + case root_data do + {_, path} -> + path - _ -> - [] - end + _ -> + [] + end - tenant = - case Enum.at(aggregates, 0) do - %{context: %{tenant: tenant}} -> - Ash.ToTenant.to_tenant(tenant, resource) + tenant = + case Enum.at(aggregates, 0) do + %{context: %{tenant: tenant}} -> + Ash.ToTenant.to_tenant(tenant, resource) - _ -> - nil - end + _ -> + nil + end - {query, aggregates} = - Enum.reduce( - aggregates, - {query, []}, - fn aggregate, {query, aggregates} -> - if is_atom(aggregate.name) do - existing_agg = query.__ash_bindings__.aggregate_defs[aggregate.name] - - if existing_agg && different_queries?(existing_agg.query, aggregate.query) do - {query, name} = use_aggregate_name(query, aggregate.name) - {query, [%{aggregate | name: name} | aggregates]} - else - {query, [aggregate | aggregates]} - end - else - {query, name} = use_aggregate_name(query, aggregate.name) + {already_computed_aggregates, remaining_aggregates} = + aggregates + |> Enum.uniq_by(& &1.name) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, root_data_path)) - {query, [%{aggregate | name: name} | aggregates]} + query = + if Enum.any?(already_computed_aggregates) && select? do + query.__ash_bindings__.bindings + |> Enum.filter(fn + {_binding, %{type: :aggregate}} -> true + _ -> false + end) + |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> + q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) + + Enum.reduce(aggs, q, fn agg, q -> + if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do + q = + update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> + [agg.name | select_aggs] + end) + + if agg.default_value do + from(row in q, + select_merge: %{ + ^agg.name => coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) + } + ) + else + from(row in q, + select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} + ) end + else + q end - ) + end) + end) + else + query + end - {already_computed_aggregates, remaining_aggregates} = - aggregates - |> Enum.uniq_by(& &1.name) - |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, [])) + query = + if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && + !query.__ash_bindings__[:lateral_join?] && + Enum.any?( + remaining_aggregates, + &(not optimizable_first_aggregate?(resource, &1, query)) + ) do + wrap_in_subquery_for_aggregates(query) + else + query + end - query = - if Enum.any?(already_computed_aggregates) && select? do - query.__ash_bindings__.bindings - |> Enum.filter(fn - {_binding, %{type: :aggregate}} -> true - _ -> false - end) - |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> - q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) - - Enum.reduce(aggs, q, fn agg, q -> - if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do - q = - update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> - [agg.name | select_aggs] - end) - - if agg.default_value do - from(row in q, - select_merge: %{ - ^agg.name => - coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) - } - ) - else - from(row in q, - select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} - ) - end - else - q - end - end) - end) - else - query - end + result = + remaining_aggregates + |> Enum.group_by(fn aggregate -> + expanded_path = + aggregate.resource + |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) + |> Enum.map(& &1.name) - query = - if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && - !query.__ash_bindings__[:lateral_join?] && - Enum.any?( - remaining_aggregates, - &(not optimizable_first_aggregate?(resource, &1, query)) - ) do - wrap_in_subquery_for_aggregates(query) - else - query - end + {expanded_path, aggregate.resource, aggregate.join_filters || %{}, + aggregate.query.action.name} + end) + |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> + {can_group, cant_group} = + Enum.split_with(aggregates, &can_group?(resource, &1, query)) - query = - if root_data_path == [] do - query - |> Map.update!(:__ash_bindings__, fn bindings -> - bindings - |> Map.update!(:aggregate_defs, fn aggregate_defs -> - Map.merge(aggregate_defs, Map.new(aggregates, &{&1.name, &1})) - end) - end) - else - query + [{{path, resource, join_filters, read_action}, can_group}] ++ + Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) + end) + |> Enum.reject(fn + {_, []} -> + true + + _ -> + false + end) + |> Enum.reduce_while( + {:ok, query, []}, + fn {{path, resource, join_filters, read_action}, aggregates}, {:ok, query, dynamics} -> + related = Ash.Resource.Info.related(resource, path) + read_action = Ash.Resource.Info.action(related, read_action) + + if read_action.modify_query do + raise """ + Data layer does not currently support aggregates over read actions that use `modify_query`. + + Resource: #{inspect(resource)} + Relationship Path: #{inspect(path)} + Action: #{read_action.name} + """ end - result = - remaining_aggregates - |> Enum.group_by(fn aggregate -> - expanded_path = - aggregate.resource - |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) - |> Enum.map(& &1.name) + {first_relationship, relationship_path} = + case path do + [] -> + {nil, []} - {expanded_path, aggregate.resource, aggregate.join_filters || %{}, - aggregate.query.action.name} - end) - |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> - {can_group, cant_group} = - Enum.split_with(aggregates, &can_group?(resource, &1, query)) + [first_relationship | rest] -> + case Ash.Resource.Info.relationship(resource, first_relationship) do + nil -> + raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" - [{{path, resource, join_filters, read_action}, can_group}] ++ - Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) - end) - |> Enum.reject(fn - {_, []} -> + first_relationship -> + if rest == [] do + {override_read_action(first_relationship, read_action.name), rest} + else + {first_relationship, rest} + end + end + end + + hydrated_agg_refs = + aggregates + |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) + |> Ash.Filter.hydrate_refs(%{ + resource: Enum.at(aggregates, 0).query.resource, + parent_stack: + if(first_relationship, do: [first_relationship.source], else: [resource]) + }) + |> elem(1) + + parent_expr = + if first_relationship do + first_relationship.filter + |> Ash.Filter.hydrate_refs(%{ + resource: first_relationship.destination, + parent_stack: [first_relationship.source] + }) + |> elem(1) + |> then(&[&1 | hydrated_agg_refs]) + |> AshSql.Join.parent_expr() + end + + used_aggregates = + Ash.Filter.used_aggregates(parent_expr, []) + + {:ok, query} = + AshSql.Aggregate.add_aggregates( + query, + used_aggregates, + resource, + false, + query.__ash_bindings__.root_binding + ) + + {:ok, query} = + AshSql.Join.join_all_relationships( + query, + parent_expr, + [], + nil, + [], + nil, + true, + nil, + nil, true + ) - _ -> - false - end) - |> Enum.reduce_while( - {:ok, query, []}, - fn {{path, resource, join_filters, read_action}, aggregates}, - {:ok, query, dynamics} -> - related = Ash.Resource.Info.related(resource, path) - read_action = Ash.Resource.Info.action(related, read_action) - - if read_action.modify_query do - raise """ - Data layer does not currently support aggregates over read actions that use `modify_query`. - - Resource: #{inspect(resource)} - Relationship Path: #{inspect(path)} - Action: #{read_action.name} - """ - end + is_single? = match?([_], aggregates) - {first_relationship, relationship_path} = - case path do - [] -> - {nil, []} - - [first_relationship | rest] -> - case Ash.Resource.Info.relationship(resource, first_relationship) do - nil -> - raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" - - first_relationship -> - if rest == [] do - {override_read_action(first_relationship, read_action.name), rest} - else - {first_relationship, rest} - end + cond do + is_single? && + optimizable_first_aggregate?( + resource, + Enum.at(aggregates, 0), + query + ) -> + case add_first_join_aggregate( + query, + resource, + hd(aggregates), + root_data, + first_relationship, + source_binding + ) do + {:ok, query, dynamic} -> + query = + if select? do + select_or_merge(query, hd(aggregates).name, dynamic) + else + query end - end - hydrated_agg_refs = - aggregates - |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) - |> Ash.Filter.hydrate_refs(%{ - resource: Enum.at(aggregates, 0).query.resource, - parent_stack: - if(first_relationship, do: [first_relationship.source], else: [resource]) - }) - |> elem(1) - - parent_expr = - if first_relationship do - first_relationship.filter - |> Ash.Filter.hydrate_refs(%{ - resource: first_relationship.destination, - parent_stack: [first_relationship.source] - }) - |> elem(1) - |> then(&[&1 | hydrated_agg_refs]) - |> AshSql.Join.parent_expr() - end + {:cont, {:ok, query, dynamics}} - used_aggregates = - Ash.Filter.used_aggregates(parent_expr, []) + {:error, error} -> + {:halt, {:error, error}} + end - {:ok, query} = - AshSql.Aggregate.add_aggregates( - query, - used_aggregates, - resource, - false, - query.__ash_bindings__.root_binding - ) + is_single? && Enum.at(aggregates, 0).kind == :exists -> + [aggregate] = aggregates - {:ok, query} = - AshSql.Join.join_all_relationships( - query, - parent_expr, - [], - nil, - [], - nil, - true, - nil, - nil, + expr = + if is_nil(Map.get(aggregate.query, :filter)) do true - ) + else + Map.get(aggregate.query, :filter) + end - is_single? = match?([_], aggregates) + {exists, acc} = + AshSql.Expr.dynamic_expr( + query, + %Ash.Query.Exists{ + path: root_data_path ++ aggregate.relationship_path, + related?: aggregate.related?, + resource: aggregate.query.resource, + expr: expr + }, + query.__ash_bindings__ + ) - cond do - is_single? && - optimizable_first_aggregate?( - resource, - Enum.at(aggregates, 0), - query - ) -> - case add_first_join_aggregate( - query, - resource, - hd(aggregates), - root_data, - first_relationship, - source_binding - ) do - {:ok, query, dynamic} -> - query = - if select? do - select_or_merge(query, hd(aggregates).name, dynamic) - else - query - end - - {:cont, {:ok, query, dynamics}} - - {:error, error} -> - {:halt, {:error, error}} - end + {:cont, + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), + [{aggregate.load, aggregate.name, exists} | dynamics]}} - is_single? && Enum.at(aggregates, 0).kind == :exists -> - [aggregate] = aggregates + true -> + tmp_query = + if first_relationship && first_relationship.type == :many_to_many do + put_in(query.__ash_bindings__[:lateral_join_bindings], [ + query.__ash_bindings__.current + ]) + |> AshSql.Bindings.explicitly_set_binding( + %{ + type: :left, + path: [first_relationship.join_relationship] + }, + query.__ash_bindings__.current + ) + else + query + end - expr = - if is_nil(Map.get(aggregate.query, :filter)) do - true - else - Map.get(aggregate.query, :filter) - end + start_bindings_at = + if first_relationship && first_relationship.type == :many_to_many do + query.__ash_bindings__.current + 1 + else + query.__ash_bindings__.current + end - {exists, acc} = - AshSql.Expr.dynamic_expr( + case get_subquery( + resource, + aggregates, + is_single?, + first_relationship, + relationship_path, + tmp_query, + start_bindings_at, + query, + source_binding, + root_data_path, + tenant, + join_filters + ) do + {:error, error} -> + {:error, error} + + {:ok, subquery} -> + query = + join_subquery( query, - %Ash.Query.Exists{ - path: root_data_path ++ aggregate.relationship_path, - related?: aggregate.related?, - resource: aggregate.query.resource, - expr: expr - }, - query.__ash_bindings__ + subquery, + first_relationship, + relationship_path, + aggregates, + source_binding, + root_data_path ) - {:cont, - {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), - [{aggregate.load, aggregate.name, exists} | dynamics]}} - - true -> - tmp_query = - if first_relationship && first_relationship.type == :many_to_many do - put_in(query.__ash_bindings__[:lateral_join_bindings], [ - query.__ash_bindings__.current - ]) - |> AshSql.Bindings.explicitly_set_binding( - %{ - type: :left, - path: [first_relationship.join_relationship] - }, - query.__ash_bindings__.current + if select? do + new_dynamics = + Enum.map( + aggregates, + &{&1.load, &1.name, + select_dynamic( + resource, + query, + &1, + query.__ash_bindings__.current - 1 + )} ) - else - query - end - start_bindings_at = - if first_relationship && first_relationship.type == :many_to_many do - query.__ash_bindings__.current + 1 - else - query.__ash_bindings__.current - end - - case get_subquery( - resource, - aggregates, - is_single?, - first_relationship, - relationship_path, - tmp_query, - start_bindings_at, - query, - source_binding, - root_data_path, - tenant, - join_filters - ) do - {:error, error} -> - {:error, error} - - {:ok, subquery} -> - query = - join_subquery( - query, - subquery, - first_relationship, - relationship_path, - aggregates, - source_binding, - root_data_path - ) - - if select? do - new_dynamics = - Enum.map( - aggregates, - &{&1.load, &1.name, - select_dynamic( - resource, - query, - &1, - query.__ash_bindings__.current - 1 - )} - ) - - {:cont, {:ok, query, new_dynamics ++ dynamics}} - else - {:cont, {:ok, query, dynamics}} - end + {:cont, {:ok, query, new_dynamics ++ dynamics}} + else + {:cont, {:ok, query, dynamics}} end end - end - ) - - case result do - {:ok, query, dynamics} -> - if select? do - {:ok, add_aggregate_selects(query, dynamics)} - else - {:ok, query} - end + end + end + ) - {:error, error} -> - {:error, error} + case result do + {:ok, query, dynamics} -> + if select? do + {:ok, add_aggregate_selects(query, dynamics)} + else + {:ok, query} end {:error, error} -> @@ -942,14 +895,6 @@ defmodule AshSql.Aggregate.Lateral do ) end - defp different_queries?(nil, nil), do: false - defp different_queries?(nil, _), do: true - defp different_queries?(_, nil), do: true - - defp different_queries?(query1, query2) do - query1.filter != query2.filter || query1.sort != query2.sort - end - @doc false def extract_shared_filters(aggregates) do aggregates @@ -1040,109 +985,6 @@ defmodule AshSql.Aggregate.Lateral do end end - defp use_aggregate_name(query, aggregate_name) do - {%{ - query - | __ash_bindings__: %{ - query.__ash_bindings__ - | current_aggregate_name: - next_aggregate_name(query.__ash_bindings__.current_aggregate_name), - aggregate_names: - Map.put( - query.__ash_bindings__.aggregate_names, - aggregate_name, - query.__ash_bindings__.current_aggregate_name - ) - } - }, query.__ash_bindings__.current_aggregate_name} - end - - defp resource_aggregates_to_aggregates(resource, query, aggregates) do - private_context = query.__ash_bindings__.context[:private] - - Enum.reduce_while(aggregates, {:ok, []}, fn - %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> - aggregate = - Ash.Actions.Read.add_calc_context( - aggregate, - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - aggregate, {:ok, aggregates} -> - resource - |> resource_aggregate_to_aggregate(aggregate, - actor: private_context[:actor], - tenant: private_context[:tenant] - ) - |> case do - {:ok, aggregate} -> - aggregate = - aggregate - |> Map.put(:load, aggregate.name) - |> Ash.Actions.Read.add_calc_context( - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - {:error, error} -> - {:halt, {:error, error}} - end - end) - end - - @doc false - def resource_aggregate_to_aggregate(resource, aggregate, opts \\ []) do - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - read_action = - aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name - - with %{valid?: true} = aggregate_query <- - Ash.Query.for_read(related, read_action, %{}, - actor: opts[:actor], - tenant: opts[:tenant] - ), - %{valid?: true} = aggregate_query <- - Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do - Ash.Query.Aggregate.new( - resource, - aggregate.name, - aggregate.kind, - path: aggregate.relationship_path, - query: aggregate_query, - field: aggregate.field, - default: aggregate.default, - filterable?: aggregate.filterable?, - type: aggregate.type, - sortable?: aggregate.filterable?, - include_nil?: aggregate.include_nil?, - constraints: aggregate.constraints, - implementation: aggregate.implementation, - uniq?: aggregate.uniq?, - read_action: read_action, - authorize?: aggregate.authorize? - ) - else - %{errors: errors} -> - {:error, errors} - end - end - defp add_first_join_aggregate( query, _resource, @@ -1191,7 +1033,7 @@ defmodule AshSql.Aggregate.Lateral do case aggregate.field do %Ash.Query.Aggregate{} = aggregate -> {:ok, ecto_query} = - add_aggregates( + AshSql.Aggregate.add_aggregates( ecto_query, [aggregate], aggregate.query.resource, @@ -1204,7 +1046,7 @@ defmodule AshSql.Aggregate.Lateral do %Ash.Resource.Aggregate{} = aggregate -> {:ok, ecto_query} = - add_aggregates( + AshSql.Aggregate.add_aggregates( ecto_query, [aggregate], Ash.Resource.Info.related(aggregate.resource, aggregate.relationship_path), @@ -1503,13 +1345,27 @@ defmodule AshSql.Aggregate.Lateral do case field do %Ash.Query.Aggregate{} = aggregate -> {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + AshSql.Aggregate.add_aggregates( + agg_query, + [aggregate], + related, + false, + source_binding, + root_data + ) agg_query %Ash.Resource.Aggregate{} = aggregate -> {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + AshSql.Aggregate.add_aggregates( + agg_query, + [aggregate], + related, + false, + source_binding, + root_data + ) agg_query @@ -1716,15 +1572,10 @@ defmodule AshSql.Aggregate.Lateral do ) end - def next_aggregate_name(i) do - @next_aggregate_names[i] || - raise Ash.Error.Framework.AssumptionFailed, - message: """ - All 1000 static names for aggregates have been used in a single query. - Congratulations, this means that you have gone so wildly beyond our imagination - of how much can fit into a single quer. Please file an issue and we will raise the limit. - """ - end + defdelegate next_aggregate_name(index), to: AshSql.Aggregate.Common + + defdelegate resource_aggregate_to_aggregate(resource, aggregate, opts \\ []), + to: AshSql.Aggregate.Common defp select_all_aggregates( aggregates, @@ -2608,7 +2459,7 @@ defmodule AshSql.Aggregate.Lateral do ) {:ok, query} = - add_aggregates( + AshSql.Aggregate.add_aggregates( query, used_aggregates, query.__ash_bindings__.resource, diff --git a/lib/expr.ex b/lib/expr.ex index c1c19d1..5936827 100644 --- a/lib/expr.ex +++ b/lib/expr.ex @@ -2341,13 +2341,13 @@ defmodule AshSql.Expr do ) do %{attribute: aggregate} = ref = - case bindings.aggregate_names[aggregate.name] do - nil -> - ref - - name -> - %{ref | attribute: %{aggregate | name: name}} - end + %{ + ref + | attribute: %{ + aggregate + | name: AshSql.Aggregate.Common.name_for(aggregate, bindings, ref.relationship_path) + } + } related? = Map.get(aggregate, :related?, true) diff --git a/test/aggregate_test.exs b/test/aggregate_test.exs index 691f6a9..93f2c76 100644 --- a/test/aggregate_test.exs +++ b/test/aggregate_test.exs @@ -79,4 +79,59 @@ defmodule AshSql.AggregateTest do assert aggregate.read_action == :read_all end end + + describe "shared aggregate normalization" do + test "reuses a string name after a different definition was requested" do + {:ok, original} = build(actor: %{id: Ash.UUID.generate()}) + original = %{original | name: "highest_score"} + filtered = %{original | query: Ash.Query.do_filter(original.query, score: 10)} + + {:ok, query, [first]} = normalize([original]) + {:ok, query, [second]} = normalize([filtered], query) + {:ok, query, [again]} = normalize([original], query) + + assert is_atom(first.name) + refute first.name == second.name + assert again.name == first.name + assert AshSql.Aggregate.Common.name_for(original, query.__ash_bindings__, []) == first.name + assert AshSql.Aggregate.Common.name_for(filtered, query.__ash_bindings__, []) == second.name + end + + test "resolves aliases independently for each attachment path" do + {:ok, original} = build(actor: %{id: Ash.UUID.generate()}) + original = %{original | name: "highest_score"} + filtered = %{original | query: Ash.Query.do_filter(original.query, score: 10)} + + {:ok, query, [first]} = normalize([original], nil, {Post, [:first]}) + {:ok, query, [second]} = normalize([filtered], query, {Post, [:second]}) + + refute first.name == second.name + + assert AshSql.Aggregate.Common.name_for(original, query.__ash_bindings__, [:first]) == + first.name + + assert AshSql.Aggregate.Common.name_for(filtered, query.__ash_bindings__, [:second]) == + second.name + end + + test "resource aggregates retain the actor and tenant during normalization" do + aggregate = Ash.Resource.Info.aggregate(Post, :highest_score) + actor = %{id: Ash.UUID.generate()} + + query = + AshSql.Bindings.default_bindings(%Ecto.Query{}, Post, __MODULE__, %{ + private: %{actor: actor, tenant: "acme"} + }) + + assert {:ok, _, [normalized]} = normalize([aggregate], query) + assert normalized.load == :highest_score + assert normalized.query.context.private.actor == actor + assert normalized.query.tenant == "acme" + end + end + + defp normalize(aggregates, query \\ nil, root_data \\ nil) do + query = query || AshSql.Bindings.default_bindings(%Ecto.Query{}, Post, __MODULE__) + AshSql.Aggregate.Common.normalize(query, aggregates, Post, root_data) + end end From 31e97054901f8ad2c48746bcf5401fbdc1951aa7 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Fri, 18 Sep 2026 22:45:50 -0700 Subject: [PATCH 08/14] fix(aggregates): preserve relationship scope and scalar types Grouped aggregate inputs skipped parts of relationship query preparation. Intermediate and join resources could lose actor, tenant and shared context, ignore configured read actions, or omit required action arguments. Missing attribute-tenant predicates could also count rows belonging to other tenants. Extract the existing relationship preparation boundary for both strategies. Preserve prepared endpoint metadata, apply tenant scope and bypass precedence throughout the path, and group only inputs with matching preparation context. Keep propagated overrides local so sibling aggregates retain their own scope. First/list aggregate filters previously ran before relationship limits and could select rows outside the bounded relationship. Apply read-action scope before bounds and aggregate filters afterward, matching scalar aggregates. Loaded scalar values and defaults also bypassed their declared constrained type. Apply the existing type conversion so custom loaders receive both. Validation: 24 focused SQLite regressions pass, including reproduced failures and controls. Full suites pass: AshSQL 11 tests, SQLite 274 tests, PostgreSQL 2 doctests, 2 properties and 868 tests (21 excluded). Extra-high independent review found no remaining blockers. Existing Sobelow and SQLite test-helper Dialyzer findings remain; corrected local REUSE checks and other gates pass. --- CHANGELOG.md | 6 ++ lib/aggregate/grouped.ex | 144 ++++++++++++++++++--------------------- lib/join.ex | 39 +++++++++-- 3 files changed, 108 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e4b23c..4d7fe75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ See [Conventional Commits](Https://conventionalcommits.org) for commit guideline * add grouped query aggregate support for root SQLite-compatible aggregate kinds * declare `:jason` as a direct dependency for grouped list aggregate defaults +### Bug Fixes: + +* share relationship query preparation between aggregate strategies, preserving context, read action arguments, and attribute tenant filters +* apply grouped aggregate filters after relationship limits while retaining read action scope before those limits +* load grouped scalar aggregate values and defaults using their declared type and constraints + ## [v0.7.6](https://github.com/ash-project/ash_sql/compare/v0.7.5...v0.7.6) (2026-09-18) diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex index 0896338..3d4534f 100644 --- a/lib/aggregate/grouped.ex +++ b/lib/aggregate/grouped.ex @@ -128,25 +128,29 @@ defmodule AshSql.Aggregate.Grouped do defp aggregate_group_key(aggregate) do read_action = (aggregate.query.action && aggregate.query.action.name) || aggregate.read_action + preparation_key = + {read_action, aggregate.query.arguments, aggregate.query.context, aggregate.query.tenant, + aggregate.query.domain} + relationship_key = case aggregate do %{related?: false, query: %{resource: resource}} -> {:unrelated, resource} %{relationship_path: relationship_path} -> {:related, relationship_path} end - {relationship_key, read_action, aggregate.join_filters || %{}, + {relationship_key, preparation_key, aggregate.join_filters || %{}, aggregate_filter_group_key(aggregate), aggregate_kind_group_key(aggregate)} end defp aggregate_relationship_path( - {{:related, relationship_path}, _read_action, _join_filters, _aggregate_filter_group, + {{:related, relationship_path}, _preparation, _join_filters, _aggregate_filter_group, _kind_group} ) do relationship_path end defp aggregate_relationship_path( - {{:unrelated, _resource}, _read_action, _join_filters, _aggregate_filter_group, + {{:unrelated, _resource}, _preparation, _join_filters, _aggregate_filter_group, _kind_group} ) do [] @@ -436,10 +440,11 @@ defmodule AshSql.Aggregate.Grouped do context ) do binding = query.__ash_bindings__.current + input_query = aggregate_input_query(query, hd(aggregates)) with :ok <- validate_aggregate_filters(aggregates), {:ok, aggregate_query} <- - aggregate_query(query, relationships, aggregates, binding) do + aggregate_query(input_query, relationships, aggregates, binding) do aggregate_query = Ecto.Query.subquery(aggregate_query) source_binding = context.source_binding @@ -469,6 +474,26 @@ defmodule AshSql.Aggregate.Grouped do end end + defp aggregate_input_query(query, aggregate) do + shared_context = aggregate.query.context[:shared] || %{} + + shared_context = + if AshSql.Join.context_multitenancy(aggregate.query) == :bypass_all do + # Ash may clear the shared wrapper after merging it into private context. + Ash.Helpers.deep_merge_maps(shared_context, %{private: %{multitenancy: :bypass_all}}) + else + shared_context + end + + # Shared aggregate context applies along the whole path, without changing + # the parent query or the context used to build sibling aggregates. + update_in(query.__ash_bindings__.context, fn context -> + context + |> Ash.Helpers.deep_merge_maps(%{shared: shared_context}) + |> Ash.Helpers.deep_merge_maps(shared_context) + end) + end + defp aggregate_query(parent_query, [relationship], [%{kind: kind} = aggregate], binding) when kind in @window_aggregate_kinds do case relationship do @@ -870,52 +895,36 @@ defmodule AshSql.Aggregate.Grouped do end defp related_query(parent_query, relationship, aggregate, binding, relationship_path) do - aggregate.query - |> Ash.Query.unset([:filter, :sort, :distinct, :select, :limit, :offset]) - |> Ash.Query.set_context(relationship.context) - |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) - |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) - |> Ash.Query.set_context(%{ - data_layer: %{ - start_bindings_at: binding, - parent_bindings: parent_query.__ash_bindings__ - } - }) - |> Ash.Query.data_layer_query(run_return_query?: false) - |> case do - {:ok, query} -> - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> limit_relationship_rows(relationship) + read_action = (aggregate.query.action && aggregate.query.action.name) || aggregate.read_action - {:error, error} -> - {:error, error} + arguments = + if relationship.read_action == read_action do + Map.merge(relationship.read_action_arguments || %{}, aggregate.query.arguments) + else + aggregate.query.arguments + end + + relationship = %{relationship | read_action: read_action, read_action_arguments: arguments} + + with {:ok, query} <- + prepared_relationship_query(parent_query, relationship, binding, + source_query: aggregate.query, + apply_filter: join_filter(aggregate, relationship_path) + ) do + limit_relationship_rows(query, relationship) end end defp related_window_query(parent_query, relationship, aggregate, binding, relationship_path) do - aggregate.query - |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) - |> Ash.Query.set_context(relationship.context) - |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) - |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) - |> Ash.Query.set_context(%{ - data_layer: %{ - start_bindings_at: binding, - parent_bindings: parent_query.__ash_bindings__ - } - }) - |> Ash.Query.data_layer_query(run_return_query?: false) - |> case do - {:ok, query} -> - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> limit_relationship_rows(relationship) - - {:error, error} -> - {:error, error} + with {:ok, query} <- + related_query(parent_query, relationship, aggregate, binding, relationship_path) do + # Aggregate filters apply to the bounded relationship, not to the rows + # used to choose that relationship's limit/offset window. + case aggregate.query.filter do + nil -> {:ok, query} + %{expression: nil} -> {:ok, query} + filter -> AshSql.Filter.filter(query, filter, relationship.destination) + end end end @@ -1039,32 +1048,11 @@ defmodule AshSql.Aggregate.Grouped do end defp intermediate_query(parent_query, relationship, binding, aggregate, relationship_path) do - read_action = - relationship.read_action || - Ash.Resource.Info.primary_action!(relationship.destination, :read).name - - relationship.destination - |> Ash.Query.for_read(read_action) - |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) - |> Ash.Query.set_context(relationship.context) - |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) - |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) - |> Ash.Query.set_context(%{ - data_layer: %{ - start_bindings_at: binding, - parent_bindings: parent_query.__ash_bindings__ - } - }) - |> Ash.Query.data_layer_query(run_return_query?: false) - |> case do - {:ok, query} -> - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> limit_relationship_rows(relationship) - - {:error, error} -> - {:error, error} + with {:ok, query} <- + prepared_relationship_query(parent_query, relationship, binding, + apply_filter: join_filter(aggregate, relationship_path) + ) do + limit_relationship_rows(query, relationship) end end @@ -1072,16 +1060,19 @@ defmodule AshSql.Aggregate.Grouped do join_relationship = Ash.Resource.Info.relationship(relationship.source, relationship.join_relationship) - relationship.through - |> Ash.Query.new() + prepared_relationship_query(parent_query, join_relationship, binding) + end + + defp prepared_relationship_query(parent_query, relationship, binding, opts \\ []) do + relationship + |> AshSql.Join.related_ash_query(parent_query, Keyword.put(opts, :start_bindings_at, binding)) + |> Ash.Query.unset(:sort) |> Ash.Query.set_context(%{ data_layer: %{ start_bindings_at: binding, parent_bindings: parent_query.__ash_bindings__ } }) - |> Ash.Query.set_context(join_relationship.context) - |> Ash.Query.do_filter(join_relationship.filter) |> Ash.Query.data_layer_query(run_return_query?: false) |> case do {:ok, query} -> @@ -1682,10 +1673,11 @@ defmodule AshSql.Aggregate.Grouped do end) end - defp loaded_aggregate_dynamic(aggregate, binding, _sql_behaviour) do + defp loaded_aggregate_dynamic(aggregate, binding, sql_behaviour) do aggregate |> loaded_aggregate_field(binding) |> maybe_default_loaded_aggregate(aggregate) + |> then(&maybe_type_dynamic(sql_behaviour, &1, aggregate)) end defp maybe_default_loaded_aggregate(dynamic, %{default_value: nil}), do: dynamic diff --git a/lib/join.ex b/lib/join.ex index d8a1893..b797e49 100644 --- a/lib/join.ex +++ b/lib/join.ex @@ -434,17 +434,18 @@ defmodule AshSql.Join do end end - defp related_query(relationship, query, opts) do - sort? = Keyword.get(opts, :sort?, false) + @doc false + def related_ash_query(relationship, query, opts \\ []) do filter = Keyword.get(opts, :filter, nil) filter_subquery? = Keyword.get(opts, :filter_subquery?, false) parent_resources = Keyword.get(opts, :parent_stack, [relationship.source]) + source_query = opts[:source_query] read_action = get_read_action(relationship) context = Map.delete(query.__ash_bindings__.context, :data_layer) - tenant = query.__ash_bindings__.context[:private][:tenant] + tenant = (source_query && source_query.tenant) || context[:private][:tenant] relationship.destination |> Ash.Query.new() @@ -456,6 +457,17 @@ defmodule AshSql.Join do start_bindings_at: opts[:start_bindings_at] || 0 } }) + |> then(fn query -> + if source_query do + # Rebuild the relationship's action scope on a fresh query. The source + # aggregate query already combines action and aggregate filters. + query + |> Ash.Query.set_context(source_query.context) + |> Ash.Query.set_domain(source_query.domain || query.domain) + else + query + end + end) |> Ash.Query.set_context(relationship.context) |> Ash.Query.do_filter(relationship.filter, parent_stack: parent_resources) |> then(fn query -> @@ -476,13 +488,19 @@ defmodule AshSql.Join do read_action.name, Map.get(relationship, :read_action_arguments, %{}), actor: context[:private][:actor], - tenant: context[:private][:tenant] + tenant: tenant ) end end) |> Ash.Query.unset([:distinct, :select, :limit, :offset]) |> handle_attribute_multitenancy(tenant, read_action) |> hydrate_refs(context[:private][:actor]) + end + + defp related_query(relationship, query, opts) do + sort? = Keyword.get(opts, :sort?, false) + + related_ash_query(relationship, query, opts) |> then(fn query -> if sort? do query @@ -557,10 +575,21 @@ defmodule AshSql.Join do end end + @doc false + def context_multitenancy(query) do + case query.context do + %{private: %{multitenancy: multitenancy}} -> multitenancy + %{multitenancy: multitenancy} -> multitenancy + _ -> nil + end + end + @doc false def handle_attribute_multitenancy(query, tenant, read_action \\ nil) do + multitenancy = context_multitenancy(query) || (read_action && read_action.multitenancy) + if tenant && Ash.Resource.Info.multitenancy_strategy(query.resource) == :attribute && - (is_nil(read_action) || read_action.multitenancy not in [:bypass, :bypass_all]) do + multitenancy not in [:bypass, :bypass_all] do multitenancy_attribute = Ash.Resource.Info.multitenancy_attribute(query.resource) if multitenancy_attribute do From f9fd7fba430909608364ac27b512f393fa7c6bf3 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:22:01 -0700 Subject: [PATCH 09/14] fix(aggregates): reject grouped distinct counts without a single key Fieldless grouped counts that need distinct records counted only the first primary-key column, or the relationship join attribute when the destination had no primary key. Composite-key and keyless destinations therefore collapsed to one row per parent. Return the same unsupported-key-shape error as grouped query aggregates. Exists aggregates no longer ask for distinct records, since row multiplication cannot change whether any row matched. --- lib/aggregate/grouped.ex | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex index 3d4534f..0cf57f2 100644 --- a/lib/aggregate/grouped.ex +++ b/lib/aggregate/grouped.ex @@ -1563,9 +1563,9 @@ defmodule AshSql.Aggregate.Grouped do defp count_dynamic(query, relationship, %{field: nil} = aggregate, binding) do if count_distinct?(aggregate) do - count_field = fieldless_count_field(relationship) - - {:ok, query, Ecto.Query.dynamic(count(field(as(^binding), ^count_field), :distinct))} + with {:ok, count_field} <- fieldless_count_field(relationship, aggregate) do + {:ok, query, Ecto.Query.dynamic(count(field(as(^binding), ^count_field), :distinct))} + end else {:ok, query, Ecto.Query.dynamic(count())} end @@ -1585,16 +1585,25 @@ defmodule AshSql.Aggregate.Grouped do end end - defp fieldless_count_field(relationship) do - relationship.destination - |> Ash.Resource.Info.primary_key() - |> List.first() - |> case do - nil -> relationship.destination_attribute - field -> field + # Distinct records are only identified by a single-column primary key. One + # column of a composite key, or the join attribute, would undercount. + defp fieldless_count_field(%{destination: destination}, aggregate) do + case Ash.Resource.Info.primary_key(destination) do + [field] -> + {:ok, field} + + [] -> + {:error, + "AshSql grouped aggregate #{inspect(aggregate.name)} requires a single primary key to count distinct records, but #{inspect(destination)} has no primary key"} + + fields -> + {:error, + "AshSql grouped aggregate #{inspect(aggregate.name)} requires a single primary key to count distinct records, but #{inspect(destination)} has composite primary key #{inspect(fields)}"} end end + # Row multiplication cannot change whether any row matched. + defp count_distinct?(%{kind: :exists}), do: false defp count_distinct?(%{uniq?: true}), do: true defp count_distinct?(%{field: nil} = aggregate) do From 9091f02759ebfd408090a0f570e984d3739c3478 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:23:06 -0700 Subject: [PATCH 10/14] fix(aggregates): reselect lateral aggregates from their attachment path Aggregate names are scoped to an attachment path, so one name can be bound at the root and at a related path in the same query. Already-added lateral aggregates were reselected from every aggregate binding with a matching name, letting a later binding overwrite the requested value. Only reselect from bindings at the requested path, the same criterion already_added?/3 uses to find them. --- lib/aggregate/lateral.ex | 2 +- test/aggregate_test.exs | 54 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/lib/aggregate/lateral.ex b/lib/aggregate/lateral.ex index cbb576c..8dcab7e 100644 --- a/lib/aggregate/lateral.ex +++ b/lib/aggregate/lateral.ex @@ -47,7 +47,7 @@ defmodule AshSql.Aggregate.Lateral do if Enum.any?(already_computed_aggregates) && select? do query.__ash_bindings__.bindings |> Enum.filter(fn - {_binding, %{type: :aggregate}} -> true + {_binding, %{type: :aggregate, path: ^root_data_path}} -> true _ -> false end) |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> diff --git a/test/aggregate_test.exs b/test/aggregate_test.exs index 93f2c76..1334629 100644 --- a/test/aggregate_test.exs +++ b/test/aggregate_test.exs @@ -5,6 +5,8 @@ defmodule AshSql.AggregateTest do use ExUnit.Case, async: true + require Ecto.Query + defmodule Comment do use Ash.Resource, domain: AshSql.AggregateTest.Domain, data_layer: Ash.DataLayer.Ets @@ -130,6 +132,58 @@ defmodule AshSql.AggregateTest do end end + describe "lateral aggregate reselection" do + setup do + {:ok, aggregate} = Ash.Query.Aggregate.new(Post, :same_name, :count, path: [:comments]) + + # Normalization scopes names by attachment path, so the same name can be + # bound at the root and at a related path in one query. + query = + Ecto.Query.from(row in "posts", as: ^0, select: %{}) + |> AshSql.Bindings.default_bindings(Post, __MODULE__) + |> AshSql.Bindings.add_binding(%{type: :aggregate, path: [], aggregates: [aggregate]}) + |> AshSql.Bindings.add_binding(%{ + type: :aggregate, + path: [:related], + aggregates: [aggregate] + }) + + %{aggregate: aggregate, query: query} + end + + test "selects a root aggregate only from the root binding", context do + {:ok, query} = + AshSql.Aggregate.Lateral.add_aggregates(context.query, [context.aggregate], Post, true, 0) + + assert selected_bindings(query) == [1] + end + + test "selects a related aggregate only from its attachment path", context do + {:ok, query} = + AshSql.Aggregate.Lateral.add_aggregates( + context.query, + [context.aggregate], + Post, + true, + 0, + {Post, [:related]} + ) + + assert selected_bindings(query) == [2] + end + end + + defp selected_bindings(query) do + query.select.expr + |> Macro.prewalk([], fn + {:as, _, [binding]} = ast, bindings -> {ast, [binding | bindings]} + ast, bindings -> {ast, bindings} + end) + |> elem(1) + |> Enum.uniq() + |> Enum.sort() + end + defp normalize(aggregates, query \\ nil, root_data \\ nil) do query = query || AshSql.Bindings.default_bindings(%Ecto.Query{}, Post, __MODULE__) AshSql.Aggregate.Common.normalize(query, aggregates, Post, root_data) From 65cceedd090c93ef07b586bd4fe0ab042fc0d041 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:24:43 -0700 Subject: [PATCH 11/14] test(aggregates): cover context-first attribute tenancy handle_attribute_multitenancy/3 now reads context multitenancy before the read action, which also changes lateral and exists callers that pass no read action. Pin that precedence against Ash.Actions.Read, which resolves read multitenancy the same way, including enforcing contexts that override a bypassing action. --- test/join_test.exs | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/test/join_test.exs b/test/join_test.exs index 8e298f9..3d81815 100644 --- a/test/join_test.exs +++ b/test/join_test.exs @@ -7,6 +7,83 @@ defmodule AshSql.JoinTest do alias AshSql.Join + defmodule Tenanted do + use Ash.Resource, domain: AshSql.JoinTest.Domain, data_layer: Ash.DataLayer.Ets + + attributes do + uuid_primary_key(:id) + attribute(:tenant_id, :string) + end + + multitenancy do + strategy(:attribute) + attribute(:tenant_id) + end + + actions do + defaults([:read]) + + read :bypass do + multitenancy(:bypass) + end + end + end + + defmodule Domain do + use Ash.Domain, validate_config_inclusion?: false + + resources do + resource(Tenanted) + end + end + + describe "handle_attribute_multitenancy/3" do + # Ash resolves read multitenancy context-first, falling back to the action. + for {context, action, filtered?} <- [ + {nil, :read, true}, + {nil, :bypass, false}, + {:allow_global, :bypass, true}, + {:enforce, :bypass, true}, + {:bypass_all, :read, false} + ] do + test "#{inspect(context)} context with the #{action} action matches Ash" do + query = tenanted_query(unquote(context), unquote(action)) + + {:ok, ash_query} = Ash.Actions.Read.handle_multitenancy(query) + sql_query = Join.handle_attribute_multitenancy(query, "acme", query.action) + + assert filtered?(sql_query) == unquote(filtered?) + assert filtered?(sql_query) == filtered?(ash_query) + end + end + + test "a bypass_all context applies without a read action" do + query = tenanted_query(:bypass_all, :read) + + refute filtered?(Join.handle_attribute_multitenancy(query, "acme")) + end + + test "the tenant filter applies without a read action or context" do + query = tenanted_query(nil, :read) + + assert filtered?(Join.handle_attribute_multitenancy(query, "acme")) + end + end + + defp tenanted_query(context, action) do + Tenanted + |> then(fn query -> + if context do + Ash.Query.set_context(query, %{private: %{multitenancy: context}}) + else + Ash.Query.new(query) + end + end) + |> Ash.Query.for_read(action, %{}, tenant: "acme") + end + + defp filtered?(query), do: not is_nil(query.filter) + defp query(data_layer_context \\ %{}) do %{__ash_bindings__: %{context: %{data_layer: data_layer_context}}} end From 1333a82c5c496b342603699eb0bfc802b3e41427 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:25:36 -0700 Subject: [PATCH 12/14] refactor(aggregates): remove unused grouped sort helpers Grouped.add_aggregates/4, add_sort_aggregates/3 and the sort-derived aggregate helpers behind them have no callers; adapters reach grouped planning through the AshSql.Aggregate facade. The unused chain also built aggregate queries without actor or tenant, so leaving it risked reuse. --- lib/aggregate/grouped.ex | 157 --------------------------------------- 1 file changed, 157 deletions(-) diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex index 0cf57f2..c051275 100644 --- a/lib/aggregate/grouped.ex +++ b/lib/aggregate/grouped.ex @@ -25,25 +25,6 @@ defmodule AshSql.Aggregate.Grouped do }) end - def add_aggregates(query, aggregates, resource, opts \\ []) do - AshSql.Aggregate.add_aggregates( - query, - aggregates, - resource, - Keyword.get(opts, :select?, true), - query.__ash_bindings__.root_binding, - nil - ) - end - - def add_sort_aggregates(query, sort, _resource) when sort in [nil, []], do: {:ok, query} - - def add_sort_aggregates(query, sort, resource) do - with {:ok, aggregates} <- aggregates_from_sort(query, sort, resource) do - add_aggregates(query, aggregates, resource, select?: false) - end - end - def relationship_filter_uses_parent?(%{filter: nil}), do: false def relationship_filter_uses_parent?(%{filter: filter}) do @@ -194,144 +175,6 @@ defmodule AshSql.Aggregate.Grouped do loaded_aggregate_dynamic(aggregate, binding, bindings.sql_behaviour)} end - defp aggregates_from_sort(query, sort, resource) do - sort - |> List.wrap() - |> Enum.reduce_while({:ok, []}, fn sort, {:ok, aggregates} -> - case sort_aggregates(query, sort, resource) do - {:ok, new_aggregates} -> - {:cont, {:ok, new_aggregates ++ aggregates}} - - {:error, error} -> - {:halt, {:error, error}} - end - end) - |> case do - {:ok, aggregates} -> {:ok, Enum.uniq(aggregates)} - {:error, error} -> {:error, error} - end - end - - defp sort_aggregates(query, {sort, _order}, resource) do - sort_key_aggregates(query, sort, resource) - end - - defp sort_aggregates(query, sort, resource) do - sort_key_aggregates(query, sort, resource) - end - - defp sort_key_aggregates(_query, %Ash.Query.Aggregate{} = aggregate, _resource) do - {:ok, [aggregate]} - end - - defp sort_key_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do - calculation_aggregates(query, calculation, resource) - end - - defp sort_key_aggregates(query, sort, resource) when is_atom(sort) do - case Ash.Resource.Info.field(resource, sort) do - %Ash.Resource.Aggregate{} = aggregate -> - query_aggregate(resource, aggregate) - - %Ash.Resource.Calculation{} = calculation -> - calculation_aggregates(query, calculation, resource) - - _ -> - {:ok, []} - end - end - - defp sort_key_aggregates(_query, _sort, _resource), do: {:ok, []} - - defp calculation_aggregates(query, %Ash.Resource.Calculation{} = calculation, resource) do - {module, opts} = calculation.calculation - - with {:ok, calculation} <- - Ash.Query.Calculation.new( - calculation.name, - module, - opts, - calculation.type, - calculation.constraints - ) do - calculation = - Ash.Actions.Read.add_calc_context( - calculation, - query.__ash_bindings__.context[:private][:actor], - query.__ash_bindings__.context[:private][:authorize?], - query.__ash_bindings__.context[:private][:tenant], - query.__ash_bindings__.context[:private][:tracer], - query.__ash_bindings__.context[:private][:domain], - query.__ash_bindings__.context[:private][:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - calculation_aggregates(query, calculation, resource) - end - end - - defp calculation_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do - calculation.opts - |> calculation.module.expression(calculation.context) - |> Ash.Filter.hydrate_refs(%{ - resource: resource, - aggregates: %{}, - parent_stack: query.__ash_bindings__[:parent_resources] || [], - calculations: %{}, - public?: false - }) - |> case do - {:ok, expression} -> - {:ok, Ash.Filter.used_aggregates(expression)} - - {:error, error} -> - {:error, error} - end - end - - defp query_aggregate(resource, aggregate) do - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - read_action = - aggregate.read_action || - Ash.Resource.Info.primary_action!(related, :read).name - - with %{valid?: true} = aggregate_query <- Ash.Query.for_read(related, read_action), - %{valid?: true} = aggregate_query <- - Ash.Query.build(aggregate_query, - filter: aggregate.filter, - sort: aggregate.sort - ), - {:ok, aggregate} <- - Ash.Query.Aggregate.new( - resource, - aggregate.name, - aggregate.kind, - path: aggregate.relationship_path, - query: aggregate_query, - field: aggregate.field, - default: aggregate.default, - filterable?: aggregate.filterable?, - type: aggregate.type, - sortable?: aggregate.sortable?, - include_nil?: aggregate.include_nil?, - constraints: aggregate.constraints, - implementation: aggregate.implementation, - uniq?: aggregate.uniq?, - read_action: read_action, - authorize?: aggregate.authorize?, - join_filters: aggregate.join_filters - ) do - {:ok, [aggregate]} - else - %{errors: errors} -> - {:error, errors} - - {:error, error} -> - {:error, error} - end - end - defp add_aggregate_group(query, _resource, [], aggregates, context) do if Enum.all?(aggregates, &(&1.related? == false)) do do_add_unrelated_aggregate_group(query, aggregates, context) From 038663556c59610c4bc01b1f03a8eee06f26b8aa Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:26:10 -0700 Subject: [PATCH 13/14] docs(aggregates): document the grouped adapter contract grouped_list_aggregate/2 must apply the planner's named window itself, and its result must be a JSON list because grouped list defaults are JSON-encoded. Grouped query aggregates also use SQLite's LIMIT -1 for offset-only input. Document these assumptions on the callbacks and in the README rather than implying a dialect-neutral strategy. --- README.md | 9 ++++++--- lib/implementation.ex | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0eae2ec..a15c719 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,12 @@ layers can override `aggregate_strategy/1` with `:grouped` when they need the SQLite-style grouped aggregate implementation. The grouped strategy uses adapter-provided list aggregation. Implementations -that select `:grouped` must implement `grouped_list_aggregate/2` and return the -windowed list expression for their SQL dialect. AshSQLite uses SQLite's JSON -list representation for this callback. +that select `:grouped` must implement `grouped_list_aggregate/2` and return a +list expression over the `:ash_sql_grouped_aggregate_window` window. The +grouped strategy assumes SQLite-compatible SQL: list defaults are JSON-encoded, +so list values must use a JSON list representation, and offset-only query +aggregates use `LIMIT -1`. AshSQLite uses SQLite's JSON list representation +for this callback. The aggregate facade normalizes resource aggregates and SQL aliases before dispatch. Both strategies preserve the source binding and attachment path, so diff --git a/lib/implementation.ex b/lib/implementation.ex index d15e9d0..b818d8a 100644 --- a/lib/implementation.ex +++ b/lib/implementation.ex @@ -42,6 +42,24 @@ defmodule AshSql.Implementation do {list(term), term} | list(term) @callback list_aggregate(Ash.Resource.t()) :: String.t() | nil + + @doc """ + Build the value of a `:list` aggregate for the `:grouped` aggregate strategy. + + `field` is a dynamic for the listed value. The grouped strategy selects the returned + expression from a window partitioned by parent and ordered by the aggregate sort, then + keeps each partition's last row. The expression must apply that window itself, by name: + + Ecto.Query.dynamic( + over(fragment("json_group_array(?)", ^field), :ash_sql_grouped_aggregate_window) + ) + + When `include_nil?` is false, nil values have already been removed from the input. + + List defaults are JSON-encoded before they are cast with `type_expr/2`, so the result must + use a JSON list representation. Returning `nil`, which is the default, makes grouped list + aggregates return an error. + """ @callback grouped_list_aggregate(term, include_nil? :: boolean) :: term | nil @callback multicolumn_distinct?() :: boolean @@ -54,6 +72,18 @@ defmodule AshSql.Implementation do @callback strpos_function() :: String.t() @callback type_expr(expr :: term, type :: term) :: term @callback ref_cast_type(type :: term) :: term + + @doc """ + Choose how aggregates over `resource` are planned. + + `:lateral`, which is the default, loads related aggregates through lateral joins. + `:grouped` joins grouped and windowed subqueries instead, for databases without lateral + joins. + + The grouped strategy assumes SQLite-compatible SQL: offset-only query aggregates use + `LIMIT -1`, and list aggregates use a JSON list representation (see + `grouped_list_aggregate/2`). + """ @callback aggregate_strategy(Ash.Resource.t()) :: :lateral | :grouped @optional_callbacks determine_types: 3 From 0985b9fdcca0a0919defdf76b0c44115fa8b8340 Mon Sep 17 00:00:00 2001 From: Will Townsend Date: Thu, 24 Sep 2026 21:26:14 -0700 Subject: [PATCH 14/14] chore: remove hand-written changelog section git_ops generates CHANGELOG.md from conventional commits at release, so the manual Unreleased section would duplicate or conflict with the generated entries. --- CHANGELOG.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d7fe75..dbe5141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,23 +11,6 @@ See [Conventional Commits](Https://conventionalcommits.org) for commit guideline -## Unreleased - -### Features: - -* add strategy-aware aggregate dispatch with lateral and grouped aggregate implementations - -### Improvements: - -* add grouped query aggregate support for root SQLite-compatible aggregate kinds -* declare `:jason` as a direct dependency for grouped list aggregate defaults - -### Bug Fixes: - -* share relationship query preparation between aggregate strategies, preserving context, read action arguments, and attribute tenant filters -* apply grouped aggregate filters after relationship limits while retaining read action scope before those limits -* load grouped scalar aggregate values and defaults using their declared type and constraints - ## [v0.7.6](https://github.com/ash-project/ash_sql/compare/v0.7.5...v0.7.6) (2026-09-18)