fix: properly hydrate and scope sorts

improvement: support anonymous aggregates and calculations in queries
This commit is contained in:
Zach Daniel 2024-05-22 17:46:39 -04:00
parent ac9afafcd9
commit 9b3eace611
13 changed files with 376 additions and 175 deletions

View file

@ -176,14 +176,7 @@ defmodule Ash.Actions.Read do
opts = Keyword.delete(opts, :page)
query = Ash.Query.page(query, page_opts)
query =
if query.page && query.page[:limit] &&
(query.page[:before] || query.page[:after] ||
(action.pagination.keyset? && !query.page[:offset])) do
load_and_select_sort(query)
else
query
end
query = load_and_select_sort(query)
pkey = Ash.Resource.Info.primary_key(query.resource)
@ -333,14 +326,6 @@ defmodule Ash.Actions.Read do
defp do_read(%{action: action} = query, calculations_at_runtime, calculations_in_query, opts) do
maybe_in_transaction(query, opts, fn ->
with {:ok, %{valid?: true} = query} <- handle_multitenancy(query),
{:ok, sort} <-
Ash.Actions.Sort.process(
query.resource,
query.sort,
query.aggregates,
query.context
),
query <- Map.put(query, :sort, sort),
query <- add_select_if_none_exists(query),
query <- %{
query
@ -363,6 +348,15 @@ defmodule Ash.Actions.Read do
calculations_at_runtime ++ calculations_in_query
),
{:ok, data_layer_calculations} <- hydrate_calculations(query, calculations_in_query),
{:ok, query} <-
hydrate_sort(
query,
opts[:actor],
opts[:authorize?],
query.tenant,
opts[:tracer],
query.domain
),
{:ok, relationship_path_filters} <-
Ash.Filter.relationship_filters(
query.domain,
@ -392,6 +386,15 @@ defmodule Ash.Actions.Read do
query.tenant,
opts[:tracer]
),
query <-
authorize_sorts(
query,
relationship_path_filters,
opts[:actor],
opts[:authorize?],
query.tenant,
opts[:tracer]
),
{:ok, filter} <-
filter_with_related(
query,
@ -968,6 +971,115 @@ defmodule Ash.Actions.Read do
end
end
defp hydrate_sort(%{sort: empty} = query, _actor, _authorize?, _tenant, _tracer, _domain)
when empty in [nil, []] do
{:ok, query}
end
defp hydrate_sort(query, actor, authorize?, tenant, tracer, domain) do
query.sort
|> List.wrap()
|> Enum.map(fn {field, direction} ->
if is_atom(field) do
case Ash.Resource.Info.field(query.resource, field) do
%Ash.Resource.Calculation{} = calc -> {calc, direction}
%Ash.Resource.Aggregate{} = agg -> {agg, direction}
_field -> {field, direction}
end
else
{field, direction}
end
end)
|> Enum.reduce_while({:ok, []}, fn
{%Ash.Resource.Calculation{} = resource_calculation, direction}, {:ok, sort} ->
{module, opts} = resource_calculation.calculation
case Ash.Query.Calculation.new(
resource_calculation.name,
module,
opts,
resource_calculation.type,
resource_calculation.constraints,
filterable?: resource_calculation.filterable?,
sortable?: resource_calculation.sortable?,
sensitive?: resource_calculation.sensitive?,
load: resource_calculation.load
) do
{:ok, calc} ->
case hydrate_calculations(query, [calc]) do
{:ok, [{calc, expression}]} ->
{:cont,
{:ok,
[
{%{
calc
| module: Ash.Resource.Calculation.Expression,
opts: [expr: expression]
}, direction}
| sort
]}}
{:error, error} ->
{:halt, {:error, error}}
end
{:error, error} ->
{:halt, {:error, error}}
end
{%Ash.Query.Calculation{} = calc, direction}, {:ok, sort} ->
case hydrate_calculations(query, [calc]) do
{:ok, [{calc, expression}]} ->
{:cont,
{:ok,
[
{%{
calc
| module: Ash.Resource.Calculation.Expression,
opts: [expr: expression]
}, direction}
| sort
]}}
{:error, error} ->
{:halt, {:error, error}}
end
{%Ash.Resource.Aggregate{} = agg, direction}, {:ok, sort} ->
case query_aggregate_from_resource_aggregate(query, agg) do
{:ok, agg} -> {:cont, {:ok, [{agg, direction} | sort]}}
{:error, error} -> {:halt, {:error, error}}
end
{other, direction}, {:ok, sort} ->
{:cont, {:ok, [{other, direction} | sort]}}
end)
|> case do
{:ok, sort} ->
sort =
Enum.map(sort, fn {field, direction} ->
case field do
%struct{} = field
when struct in [
Ash.Query.Calculation,
Ash.Aggregate.Calculation,
Ash.Resource.Calculation,
Ash.Resource.Aggregate
] ->
{add_calc_context(field, actor, authorize?, tenant, tracer, domain), direction}
field ->
{field, direction}
end
end)
{:ok, %{query | sort: Enum.reverse(sort)}}
{:error, error} ->
{:error, error}
end
end
defp compute_expression_at_runtime_for_missing_records(data, query, data_layer_calculations) do
if Enum.any?(data, & &1.__metadata__[:private][:missing_from_data_layer]) do
{require_calculating, rest} =
@ -1660,6 +1772,22 @@ defmodule Ash.Actions.Read do
{key, load}
end
end),
sort:
Enum.map(query.sort, fn {field, direction} ->
case field do
%struct{} = calc
when struct in [
Ash.Query.Calculation,
Ash.Aggregate.Calculation,
Ash.Resource.Calculation,
Ash.Resource.Aggregate
] ->
{add_calc_context(calc, actor, authorize?, tenant, tracer, domain), direction}
other ->
{other, direction}
end
end),
aggregates:
Map.new(query.aggregates, fn {key, agg} ->
{key,
@ -2125,9 +2253,6 @@ defmodule Ash.Actions.Read do
end
defp keyset_pagination(query, pagination, opts) do
query =
load_and_select_sort(query)
limited = Ash.Query.limit(query, limit(query, opts[:limit], query.limit, pagination) + 1)
if opts[:before] || opts[:after] do
@ -2171,41 +2296,18 @@ defmodule Ash.Actions.Read do
end
defp load_and_select_sort(query) do
query.sort
|> Enum.map(fn
{%Ash.Query.Calculation{} = calc, _} ->
{:calc, calc}
{field, _} ->
cond do
Ash.Resource.Info.aggregate(query.resource, field) ->
{:agg, field}
Ash.Resource.Info.attribute(query.resource, field) ->
{:attr, field}
end
end)
|> Enum.reduce(query, fn
{:calc, %{load: nil} = calc}, query ->
Ash.Query.calculate(
query,
calc.name,
calc.type,
{calc.module, calc.opts},
calc.context.arguments,
calc.constraints,
calc.context
)
{:calc, calc}, query ->
Ash.Query.load(query, calc)
{:agg, field}, query ->
Ash.Query.load(query, field)
{:attr, field}, query ->
Ash.Query.ensure_selected(query, field)
end)
query.resource
|> Ash.Resource.Info.actions()
|> Enum.any?(&match?(%{pagination: %{keyset?: true}}, &1))
|> if do
query.sort
|> Enum.map(&elem(&1, 0))
|> then(fn load ->
Ash.Query.load(query, load)
end)
else
query
end
end
defp limit(query, page_size, query_limit, pagination) do
@ -2350,7 +2452,8 @@ defmodule Ash.Actions.Read do
case Ash.Filter.hydrate_refs(expression, %{
resource: query.resource,
public?: false
public?: false,
parent_stack: parent_stack_from_context(query)
}) do
{:ok, expression} ->
{:cont, {:ok, [{calculation, expression} | calculations]}}
@ -2366,6 +2469,18 @@ defmodule Ash.Actions.Read do
end)
end
defp parent_stack_from_context(%{
context: %{
data_layer: %{lateral_join_source: {_, [{%{resource: resource}, _, _, _} | _]}}
}
}) do
[resource]
end
defp parent_stack_from_context(_query) do
[]
end
defp authorize_calculation_expressions(
hydrated_calculations,
resource,
@ -2412,6 +2527,103 @@ defmodule Ash.Actions.Read do
end)
end
defp authorize_sorts(query, path_filters, actor, authorize?, tenant, tracer) do
Enum.reduce_while(query.sort, {:ok, []}, fn
{%Ash.Query.Aggregate{} = aggregate, direction}, {:ok, sort} ->
new_agg =
if authorize? && aggregate.authorize? do
authorize_aggregate(
aggregate,
path_filters,
actor,
authorize?,
tenant,
tracer,
query.domain
)
else
add_calc_context(aggregate, actor, authorize?, tenant, tracer, query.domain)
end
{:cont, {:ok, [{new_agg, direction} | sort]}}
{%Ash.Query.Calculation{
module: Ash.Resource.Calculation.Expression,
opts: [expression: expression]
} = calc, direction},
{:ok, sort} ->
new_expr =
update_aggregate_filters(
expression,
query.resource,
authorize?,
path_filters,
actor,
tenant,
tracer,
query.domain
)
new_calc = %{calc | opts: [expression: new_expr]}
{:cont, {:ok, [{new_calc, direction} | sort]}}
{other, direction}, {:ok, sort} ->
{:cont, {:ok, [{other, direction} | sort]}}
end)
|> case do
{:ok, reversed_sort} ->
%{query | sort: Enum.reverse(reversed_sort)}
{:error, error} ->
Ash.Query.add_error(query, error)
end
end
defp query_aggregate_from_resource_aggregate(query, resource_aggregate) do
resource = query.resource
related_resource = Ash.Resource.Info.related(resource, resource_aggregate.relationship_path)
read_action =
resource_aggregate.read_action ||
Ash.Resource.Info.primary_action!(related_resource, :read).name
with %{valid?: true} = aggregate_query <-
Ash.Query.for_read(related_resource, read_action),
%{valid?: true} = aggregate_query <-
Ash.Query.Aggregate.build_query(aggregate_query,
filter: resource_aggregate.filter,
sort: resource_aggregate.sort
),
{:ok, query_aggregate} <-
Ash.Query.Aggregate.new(
resource,
resource_aggregate.name,
resource_aggregate.kind,
path: resource_aggregate.relationship_path,
query: aggregate_query,
field: resource_aggregate.field,
default: resource_aggregate.default,
filterable?: resource_aggregate.filterable?,
type: resource_aggregate.type,
constraints: resource_aggregate.constraints,
implementation: resource_aggregate.implementation,
uniq?: resource_aggregate.uniq?,
read_action: read_action,
authorize?: resource_aggregate.authorize?,
join_filters:
Map.new(resource_aggregate.join_filters, &{&1.relationship_path, &1.filter})
) do
{:ok, Map.put(query_aggregate, :load, resource_aggregate.name)}
else
{:error, error} ->
{:error, error}
%{errors: errors} ->
{:error, errors}
end
end
defp authorize_aggregate(aggregate, path_filters, actor, authorize?, tenant, tracer, domain) do
aggregate = add_calc_context(aggregate, actor, authorize?, tenant, tracer, domain)
last_relationship = last_relationship(aggregate.resource, aggregate.relationship_path)
@ -2596,6 +2808,9 @@ defmodule Ash.Actions.Read do
end
%Ash.Resource.Aggregate{} = resource_aggregate ->
related_resource =
Ash.Resource.Info.related(related_resource, aggregate.relationship_path)
read_action =
resource_aggregate.read_action ||
Ash.Resource.Info.primary_action!(related_resource, :read).name
@ -2604,8 +2819,8 @@ defmodule Ash.Actions.Read do
Ash.Query.for_read(related_resource, read_action),
%{valid?: true} = aggregate_query <-
Ash.Query.Aggregate.build_query(aggregate_query,
filter: aggregate.filter,
sort: aggregate.sort
filter: resource_aggregate.filter,
sort: resource_aggregate.sort
),
{:ok, query_aggregate} <-
Ash.Query.Aggregate.new(

View file

@ -73,56 +73,62 @@ defmodule Ash.Actions.Sort do
{%Ash.Query.Calculation{sortable?: false} = calc, _order}, {sorts, errors} ->
{sorts, [UnsortableField.exception(resource: resource, field: calc) | errors]}
{%Ash.Query.Calculation{
name: :__expr_sort__,
module: Ash.Resource.Calculation.Expression,
opts: [{:expr, expr}]
} = calc, order},
{sorts, errors} ->
expr
|> Ash.Filter.list_refs()
|> Enum.reduce_while(:ok, fn %{relationship_path: path, attribute: attribute}, :ok ->
{ref_attribute, field_name} =
case attribute do
atom when is_atom(attribute) ->
{Ash.Resource.Info.field(Ash.Resource.Info.related(resource, path), attribute),
atom}
{%Ash.Query.Calculation{} = calc, order}, {sorts, errors} ->
if String.starts_with?(to_string(calc.name), "__expr_sort__") do
%{opts: [{:expr, expr}]} = calc
%struct{} = attribute when struct in [Ash.Query.Aggregate, Ash.Query.Calculation] ->
{attribute, attribute}
expr
|> Ash.Filter.list_refs()
|> Enum.reduce_while(:ok, fn %{relationship_path: path, attribute: attribute}, :ok ->
{ref_attribute, field_name} =
case attribute do
atom when is_atom(attribute) ->
{Ash.Resource.Info.field(Ash.Resource.Info.related(resource, path), attribute),
atom}
other ->
{other, other.name}
end
%struct{} = attribute
when struct in [Ash.Query.Aggregate, Ash.Query.Calculation] ->
{attribute, attribute}
if ref_attribute.sortable? do
case find_non_sortable_relationship(resource, path, sort) do
nil ->
{:cont, :ok}
other ->
{other, other.name}
end
{resource, non_sortable_field} ->
{:halt, {:error, resource, non_sortable_field}}
end
else
{:halt, {:error, resource, field_name}}
end
end)
|> case do
:ok ->
if order in @sort_orders do
{sorts ++ [{calc, order}], errors}
if ref_attribute.sortable? do
case find_non_sortable_relationship(resource, path, sort) do
nil ->
{:cont, :ok}
{resource, non_sortable_field} ->
{:halt, {:error, resource, non_sortable_field}}
end
else
{sorts, [InvalidSortOrder.exception(order: order) | errors]}
{:halt, {:error, resource, field_name}}
end
end)
|> case do
:ok ->
if order in @sort_orders do
{sorts ++ [{calc, order}], errors}
else
{sorts, [InvalidSortOrder.exception(order: order) | errors]}
end
{:error, resource, non_sortable_field} ->
{sorts,
[UnsortableField.exception(resource: resource, field: non_sortable_field) | errors]}
{:error, resource, non_sortable_field} ->
{sorts,
[UnsortableField.exception(resource: resource, field: non_sortable_field) | errors]}
end
else
if order in @sort_orders do
{sorts ++ [{calc, order}], errors}
else
{sorts, [InvalidSortOrder.exception(order: order) | errors]}
end
end
{%Ash.Query.Calculation{} = calc, order}, {sorts, errors} ->
{%{__struct__: Ash.Query.Aggregate} = agg, order}, {sorts, errors} ->
if order in @sort_orders do
{sorts ++ [{calc, order}], errors}
{sorts ++ [{agg, order}], errors}
else
{sorts, [InvalidSortOrder.exception(order: order) | errors]}
end
@ -178,7 +184,7 @@ defmodule Ash.Actions.Sort do
end
!attribute ->
{sorts, [NoSuchField.exception(attribute: field, resource: resource) | errors]}
{sorts, [NoSuchField.exception(field: field, resource: resource) | errors]}
!attribute.sortable? ->
{sorts,
@ -371,7 +377,7 @@ defmodule Ash.Actions.Sort do
results
|> load_field(field, resource, opts)
|> Enum.sort_by(&resolve_field(&1, field, resource, domain: opts), to_sort_by_fun(direction))
|> Enum.sort_by(&resolve_field(&1, field), to_sort_by_fun(direction))
end
def runtime_sort(results, [{field, direction} | rest], opts) do
@ -379,7 +385,7 @@ defmodule Ash.Actions.Sort do
results
|> load_field(field, resource, opts)
|> Enum.group_by(&resolve_field(&1, field, resource, domain: opts))
|> Enum.group_by(&resolve_field(&1, field))
|> Enum.sort_by(fn {key, _value} -> key end, to_sort_by_fun(direction))
|> Enum.flat_map(fn {_, records} ->
runtime_sort(records, rest, Keyword.put(opts, :rekey?, false))
@ -422,7 +428,7 @@ defmodule Ash.Actions.Sort do
def runtime_distinct([%resource{} | _] = results, [{field, direction} | rest], opts) do
results
|> load_field(field, resource, opts)
|> Enum.group_by(&resolve_field(&1, field, resource, domain: opts))
|> Enum.group_by(&resolve_field(&1, field))
|> Enum.sort_by(fn {key, _value} -> key end, to_sort_by_fun(direction))
|> Enum.map(fn {_key, [first | _]} ->
first
@ -453,48 +459,20 @@ defmodule Ash.Actions.Sort do
end
end
defp resolve_field(record, %Ash.Query.Calculation{} = calc, resource, opts) do
cond do
calc.module.has_calculate?() ->
context = Map.put(calc.context, :domain, opts[:domain])
case calc.module.calculate([record], calc.opts, context) do
{:ok, [value]} -> value
_ -> nil
end
calc.module.has_expression?() ->
expression = calc.module.expression(calc.opts, calc.context)
case Ash.Filter.hydrate_refs(expression, %{
resource: resource,
aggregates: %{},
calculations: %{},
public?: false
}) do
{:ok, expression} ->
case Ash.Expr.eval_hydrated(expression, record: record, resource: resource) do
{:ok, value} ->
value
_ ->
nil
end
_ ->
nil
end
true ->
nil
end
|> case do
%Ash.ForbiddenField{} -> nil
other -> other
defp resolve_field(record, %{__struct__: struct} = agg)
when struct in [Ash.Query.Calculation, Ash.Query.Aggregate] do
if agg.load do
Map.get(record, agg.load)
else
if struct == Ash.Query.Calculation do
Map.get(record.calculations, agg.name)
else
Map.get(record.aggregates, agg.name)
end
end
end
defp resolve_field(record, field, _resource, _) do
defp resolve_field(record, field) do
record
|> Map.get(field)
|> case do

View file

@ -2844,7 +2844,8 @@ defmodule Ash.Filter do
end
end
defp add_expression_part({%Ash.Query.Calculation{} = calc, rest}, context, expression) do
defp add_expression_part({%{__struct__: field_struct} = calc, rest}, context, expression)
when field_struct in [Ash.Query.Calculation, Ash.Query.Aggregate] do
case parse_predicates(rest, calc, context) do
{:ok, nested_statement} ->
{:ok, BooleanExpression.optimized_new(:and, expression, nested_statement)}
@ -2854,7 +2855,8 @@ defmodule Ash.Filter do
end
end
defp add_expression_part(%Ash.Query.Calculation{} = calc, _context, expression) do
defp add_expression_part(%{__struct__: field_struct} = calc, _context, expression)
when field_struct in [Ash.Query.Calculation, Ash.Query.Aggregate] do
{:ok, BooleanExpression.optimized_new(:and, calc, expression)}
end

View file

@ -95,7 +95,8 @@ defmodule Ash.Page.Keyset do
field =
case field do
%Ash.Query.Calculation{} = calc ->
%{__struct__: field_struct} = calc
when field_struct in [Ash.Query.Calculation, Ash.Query.Aggregate] ->
calc
field ->
@ -199,13 +200,20 @@ defmodule Ash.Page.Keyset do
defp field_values(record, sort) do
Enum.map(sort, fn
{%Ash.Query.Calculation{load: load, name: name}, _} ->
{%{__struct__: Ash.Query.Calculation, load: load, name: name}, _} ->
if load do
Map.get(record, load)
else
Map.get(record.calculations, name)
end
{%{__struct__: Ash.Query.Aggregate, load: load, name: name}, _} ->
if load do
Map.get(record, load)
else
Map.get(record.aggregates, name)
end
{field, _} ->
Map.get(record, field)
end)

View file

@ -342,8 +342,28 @@ defmodule Ash.Query do
add_error(query, :sort, "Data layer does not support sorting")
end
end
|> sequence_expr_sorts()
end
# sobelow_skip ["DOS.BinToAtom", "DOS.StringToAtom"]
defp sequence_expr_sorts(%{sort: sort} = query) when is_list(sort) and sort != [] do
%{
query
| sort:
query.sort
|> Enum.with_index()
|> Enum.map(fn
{{%Ash.Query.Calculation{name: :__expr_sort__} = field, direction}, index} ->
{%{field | name: String.to_atom("__expr_sort__#{index}"), load: nil}, direction}
{other, _} ->
other
end)
}
end
defp sequence_expr_sorts(query), do: query
@doc """
Attach a filter statement to the query.
@ -2716,6 +2736,7 @@ defmodule Ash.Query do
add_error(query, :sort, "Data layer does not support sorting")
end
end
|> sequence_expr_sorts()
end
@doc """

View file

@ -8,29 +8,6 @@ defmodule Ash.Query.Ref do
defimpl Inspect do
def inspect(ref, _opts) do
case ref.attribute do
%Ash.Query.Calculation{} ->
case Map.drop(ref.attribute.context || %{}, [:context, :ash]) do
empty when empty == %{} ->
inspect_ref(ref)
args ->
inspect(%Ash.Query.Call{
name: ref.attribute.name,
relationship_path: ref.relationship_path,
args: [args]
})
end
%Ash.Query.Aggregate{} ->
inspect_ref(ref)
_ ->
inspect_ref(ref)
end
end
defp inspect_ref(ref) do
name =
case ref.attribute do
%{name: name} -> name

View file

@ -3,6 +3,8 @@ defmodule Ash.Query.Type do
def try_cast(value, type, constraints \\ [])
def try_cast(value, nil, _constraints), do: {:ok, value}
def try_cast(list, {:array, type}, constraints) do
if Enumerable.impl_for(list) do
list

View file

@ -39,8 +39,6 @@ defmodule Ash.Resource.Calculation.Expression do
Enum.reduce_while(records, {:ok, []}, fn record, {:ok, values} ->
case Ash.Filter.hydrate_refs(expression, %{
resource: resource,
aggregates: %{},
calculations: %{},
public?: false
}) do
{:ok, expression} ->

View file

@ -1430,7 +1430,7 @@ defmodule Ash.Resource.Dsl do
Ash.Resource.Transformers.ManyToManyDestinationAttributeOnJoinResource,
Ash.Resource.Transformers.CreateJoinRelationship,
Ash.Resource.Transformers.CachePrimaryKey,
Ash.Resource.Transformers.ValidatePrimaryActions,
Ash.Resource.Transformers.SetPrimaryActions,
Ash.Resource.Transformers.DefaultAccept,
Ash.Resource.Transformers.RequireUniqueFieldNames,
Ash.Resource.Transformers.SetDefineFor,

View file

@ -89,6 +89,6 @@ defmodule Ash.Resource.Transformers.DefaultAccept do
def after?(Ash.Resource.Transformers.BelongsToAttribute), do: true
def after?(Ash.Resource.Transformers.CreateJoinRelationship), do: true
def after?(Ash.Resource.Transformers.ValidatePrimaryActions), do: true
def after?(Ash.Resource.Transformers.SetPrimaryActions), do: true
def after?(_), do: false
end

View file

@ -29,6 +29,6 @@ defmodule Ash.Resource.Transformers.RequireUniqueActionNames do
{:ok, dsl_state}
end
def after?(Ash.Resource.Transformers.ValidatePrimaryActions), do: true
def after?(Ash.Resource.Transformers.SetPrimaryActions), do: true
def after?(_), do: false
end

View file

@ -1,4 +1,4 @@
defmodule Ash.Resource.Transformers.ValidatePrimaryActions do
defmodule Ash.Resource.Transformers.SetPrimaryActions do
@moduledoc """
Validates the primary action configuration

View file

@ -30,9 +30,9 @@ defmodule Ash.Sort do
For example:
```elixir
Ash.Query.sort(Ash.Sort.expr_sort(author.full_name, :string))
Ash.Query.sort(query, Ash.Sort.expr_sort(author.full_name, :string))
Ash.Query.sort([{Ash.Sort.expr_sort(author.full_name, :string), :desc_nils_first}])
Ash.Query.sort(query, [{Ash.Sort.expr_sort(author.full_name, :string), :desc_nils_first}])
```
"""
@spec expr_sort(Ash.Expr.t(), Ash.Type.t() | nil) :: Ash.Expr.t()