ash-project/ash
1
0
Fork 0
mirror of https://github.com/ash-project/ash.git synced 2024-09-19 21:13:10 +12:00
Code Issues Projects Releases Packages Wiki Activity

improvement: rewrite calculation loader

Browse source 
Calculation loading is complex because different calculations can
depend on differently parameterized things. FOr example:

```elixir
def load(_, _, _), do: [foo: %{arg: 1}]
def load(_, _, _), do: [foo: %{arg: 2}]
```

The previous naive implementation would simply merge all of the calculation loads, which naturally would not work. Now we ensure that we load each requirement in isolation.
This commit is contained in:
Zach Daniel 2023-03-28 20:31:47 -04:00
parent de943509f7
commit ad347ca38b
14 changed files with 1766 additions and 460 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/ash/actions/helpers.ex
View file

@ -152,7 +152,7 @@ defmodule Ash.Actions.Helpers do
if Keyword.has_key?(opts, :actor) do
Keyword.put_new(opts, :authorize?, true)
else
opts
Keyword.put(opts, :authorize?, opts[:authorize?] || Keyword.has_key?(opts, :actor))
end
end
else
@ -202,7 +202,7 @@ defmodule Ash.Actions.Helpers do
:ok
missed ->
case Application.get_env(:ash, :missed_notifications, :ignore) do
case Application.get_env(:ash, :missed_notifications, :warn) do
:ignore ->
:ok

593
lib/ash/actions/load.ex
View file

@ -407,23 +407,9 @@ defmodule Ash.Actions.Load do
end
defp do_requests(relationship, lazy?, opts, request_path, related_query, path, root_query) do
load_request =
load_request(
relationship,
lazy?,
opts,
request_path,
related_query,
root_query,
path
)
case relationship.type do
:many_to_many ->
if lateral_join?(related_query, relationship, :unknown) do
[load_request]
else
join_assoc_request =
if relationship.type == :many_to_many &&
!lateral_join?(related_query, relationship, :unknown) do
join_assoc_request(
relationship,
request_path,
@ -433,15 +419,156 @@ defmodule Ash.Actions.Load do
opts,
lazy?
)
[join_assoc_request, load_request]
end
_ ->
join_request_path =
if join_assoc_request do
join_assoc_request.path
end
load_request =
load_request(
relationship,
lazy?,
opts,
request_path,
related_query,
root_query,
path,
join_request_path
)
if join_assoc_request do
[join_assoc_request, load_request]
else
[load_request]
end
end
@doc false
def calc_dep_requests(
relationship,
lazy?,
opts,
calc_dep,
path,
root_query,
select,
load
) do
join_assoc_request =
if relationship.type == :many_to_many &&
!lateral_join?(calc_dep.query, relationship, :unknown) do
calc_dep_join_assoc_request(
relationship,
path,
root_query,
path,
opts,
lazy?,
calc_dep
)
end
load_request =
calc_dep_load_request(
relationship,
lazy?,
opts,
calc_dep,
root_query,
path,
select,
load
)
if join_assoc_request do
[join_assoc_request, load_request]
else
[load_request]
end
end
defp calc_dep_load_request(
relationship,
lazy?,
opts,
calc_dep,
root_query,
path,
select,
load
) do
{parent_dep_path, parent_data_path, parent_query_path} =
case calc_dep.path do
[] ->
{path ++ [:fetch, :data], path ++ [:fetch, :data, :results], path ++ [:fetch, :query]}
dependent_path ->
{relationship, query} = List.last(dependent_path)
depended_dep = %{
type: :relationship,
path: :lists.droplast(dependent_path),
query: query,
relationship: relationship
}
data = path ++ [:calc_dep, depended_dep, :data]
{data, data, path ++ [:calc_dep, depended_dep, :query]}
end
this_request_path = path ++ [:calc_dep, calc_dep]
dependencies = [
this_request_path ++ [:authorization_filter],
parent_dep_path,
parent_query_path
]
dependencies =
if relationship.type == :many_to_many &&
!lateral_join?(calc_dep.query, relationship, :unknown) do
dependencies ++ [this_request_path ++ [:join, :data]]
else
dependencies
end
rel_string =
calc_dep.path
|> Enum.map(&elem(&1, 0))
|> Enum.concat([calc_dep.relationship])
|> Enum.join(".")
actual_query = calc_dep.query |> Ash.Query.select(select) |> Ash.Query.load(load)
Request.new(
action:
calc_dep.query.action || Ash.Resource.Info.primary_action(relationship.destination, :read),
resource: relationship.destination,
name: "calc dep #{rel_string}",
api: calc_dep.query.api,
path: this_request_path,
query: actual_query,
data:
calc_dep_data(
relationship,
lazy?,
dependencies,
this_request_path,
calc_dep,
path,
root_query,
opts,
parent_data_path,
parent_query_path,
this_request_path ++ [:join],
actual_query
)
)
end
defp load_request(
relationship,
lazy?,
@ -449,7 +576,8 @@ defmodule Ash.Actions.Load do
request_path,
related_query,
root_query,
path
path,
join_request_path
) do
relationship_path = Enum.reverse(Enum.map([relationship | path], &Map.get(&1, :name)))
@ -521,11 +649,149 @@ defmodule Ash.Actions.Load do
related_query,
path,
root_query,
opts
opts,
join_request_path
)
)
end
defp calc_dep_data(
%{manual: manual} = relationship,
lazy?,
dependencies,
_this_request_path,
calc_dep,
_path,
root_query,
request_opts,
parent_data_path,
_parent_query_path,
_join_request_path,
actual_query
)
when not is_nil(manual) do
{mod, opts} =
case manual do
{mod, opts} ->
{mod, opts}
mod ->
{mod, []}
end
Request.resolve(dependencies, fn data ->
data = get_in(data, parent_data_path)
lazy_load_or(
data,
lazy?,
relationship.name,
root_query.query.api,
actual_query,
request_opts,
fn ->
data
|> mod.load(opts, %{
relationship: relationship,
query: actual_query,
actor: request_opts[:actor],
authorize?: request_opts[:authorize?],
api: calc_dep.query.api,
tenant: root_query.tenant
})
|> case do
{:ok, result} ->
# TODO: this will result in quite a few requests potentially for aggs/calcs
# This should be optimized.
Enum.reduce_while(result, {:ok, %{}}, fn {key, records}, {:ok, acc} ->
case calc_dep.query.api.load(records, %{actual_query | load: []},
lazy?: true,
tenant: calc_dep.query.tenant,
actor: request_opts[:actor],
authorize?: request_opts[:authorize?],
tracer: request_opts[:tracer]
) do
{:ok, results} ->
{:cont, {:ok, Map.put(acc, key, results)}}
{:error, error} ->
{:halt, {:error, error}}
end
end)
{:error, error} ->
{:error, error}
end
end
)
end)
end
defp calc_dep_data(
relationship,
lazy?,
dependencies,
this_request_path,
calc_dep,
path,
_root_query,
opts,
parent_data_path,
parent_query_path,
join_request_path,
actual_query
) do
Request.resolve(dependencies, fn data ->
base_query =
case get_in(data, this_request_path ++ [:authorization_filter]) do
nil ->
actual_query
authorization_filter ->
Ash.Query.filter(actual_query, ^authorization_filter)
end
source_query = get_in(data, parent_query_path)
source_query =
if calc_dep.query.tenant do
Ash.Query.set_tenant(source_query, calc_dep.query.tenant)
else
source_query
end
with {:ok, new_query} <-
true_load_query(
relationship,
base_query,
data,
parent_data_path,
join_request_path
),
{:ok, results} <-
run_actual_query(
new_query,
base_query,
data,
path,
relationship,
source_query,
opts,
lazy?,
parent_data_path,
join_request_path
) do
{:ok, results}
else
:nothing ->
{:ok, []}
{:error, error} ->
{:error, error}
end
end)
end
defp data(
%{manual: manual} = relationship,
lazy?,
@ -535,7 +801,8 @@ defmodule Ash.Actions.Load do
related_query,
path,
root_query,
request_opts
request_opts,
_join_request_path
)
when not is_nil(manual) do
{mod, opts} =
@ -626,7 +893,8 @@ defmodule Ash.Actions.Load do
related_query,
path,
root_query,
opts
opts,
join_request_path
) do
Request.resolve(dependencies, fn data ->
base_query =
@ -662,13 +930,31 @@ defmodule Ash.Actions.Load do
source_query
end
data_path =
if relationship.type == :many_to_many do
join_relationship = join_relationship(relationship)
join_relationship_path(path, join_relationship) |> Enum.map(& &1.name)
else
path |> Enum.reverse() |> Enum.map(& &1.name)
end
source_data_path =
case path do
[] ->
request_path ++ [:data, :results]
_path ->
request_path ++ [:load] ++ [data_path] ++ [:data]
end
with {:ok, new_query} <-
true_load_query(
relationship,
base_query,
data,
path,
request_path
source_data_path,
join_request_path
),
{:ok, results} <-
run_actual_query(
@ -678,9 +964,10 @@ defmodule Ash.Actions.Load do
path,
relationship,
source_query,
request_path,
opts,
lazy?
lazy?,
source_data_path,
join_request_path
) do
{:ok, results}
else
@ -757,22 +1044,6 @@ defmodule Ash.Actions.Load do
]
end
lateral_join? = lateral_join?(related_query, relationship, :unknown)
dependencies =
if lateral_join? do
dependencies ++
[
[
:load,
Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name],
:data
]
]
else
dependencies
end
dependencies = [[:data] | dependencies]
related_query =
@ -858,13 +1129,24 @@ defmodule Ash.Actions.Load do
source_query
end
data_path = path |> Enum.reverse() |> Enum.map(& &1.name)
source_data_path =
case path do
[] ->
request_path ++ [:data, :results]
_path ->
request_path ++ [:load] ++ [data_path] ++ [:data]
end
with {:ok, new_query} <-
true_load_query(
join_relationship,
base_query,
data,
path,
request_path
source_data_path,
nil
),
new_query <-
add_join_destination_filter(
@ -872,7 +1154,11 @@ defmodule Ash.Actions.Load do
lateral_join?,
data,
relationship,
Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name]
[
:load,
Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name],
:data
]
),
{:ok, results} <-
run_actual_query(
@ -882,9 +1168,129 @@ defmodule Ash.Actions.Load do
path,
join_relationship,
source_query,
request_path,
opts,
lazy?
lazy?,
source_data_path,
nil
) do
{:ok, results}
else
:nothing ->
{:ok, []}
{:error, error} ->
{:error, error}
end
end)
)
end
defp calc_dep_join_assoc_request(
relationship,
path,
root_query,
path,
opts,
lazy?,
calc_dep
) do
{parent_dep_path, parent_data_path, parent_query_path} =
case calc_dep.path do
[] ->
{path ++ [:fetch], path ++ [:fetch, :data, :results], path ++ [:fetch, :query]}
dependent_path ->
{relationship, query} = List.last(dependent_path)
depended_dep = %{
type: :relationship,
path: :lists.droplast(dependent_path),
query: query,
relationship: relationship
}
{path ++ [:calc_dep, depended_dep], path ++ [:calc_dep, depended_dep, :data, :results],
path ++ [:calc_dep, depended_dep, :query]}
end
this_request_path = path ++ [:calc_dep, calc_dep]
join_relationship = join_relationship(relationship)
dependencies = [
this_request_path ++ [:authorization_filter],
parent_dep_path ++ [:data],
parent_dep_path ++ [:query]
]
base_query =
join_relationship.destination
|> Ash.Query.new(join_relationship.api || root_query.api)
|> Ash.Query.set_tenant(calc_dep.query.tenant)
|> Ash.Query.set_context(join_relationship.context)
Request.new(
action:
calc_dep.query.action ||
Ash.Resource.Info.primary_action(join_relationship.destination, :read),
resource: relationship.through,
name: "load calc_dep join #{join_relationship.name}",
api: calc_dep.query.api,
path: path ++ [:calc_dep, calc_dep, :join],
query: base_query,
data:
Request.resolve(dependencies, fn
data ->
base_query =
case get_in(
data,
this_request_path ++ [:authorization_filter]
) do
nil ->
base_query
authorization_filter ->
base_query
|> Ash.Query.do_filter(authorization_filter)
end
source_data = get_in(data, parent_data_path)
lateral_join? = lateral_join?(calc_dep.query, relationship, source_data)
source_query =
get_in(
data,
parent_query_path
)
with {:ok, new_query} <-
true_load_query(
join_relationship,
base_query,
data,
parent_data_path,
nil
),
new_query <-
add_join_destination_filter(
new_query,
lateral_join?,
data,
relationship,
parent_data_path
),
{:ok, results} <-
run_actual_query(
new_query,
base_query,
data,
path,
relationship,
source_query,
opts,
lazy?,
parent_data_path,
nil
) do
{:ok, results}
else
@ -901,11 +1307,7 @@ defmodule Ash.Actions.Load do
defp add_join_destination_filter(query, true, data, relationship, destination_path) do
ids =
data
|> get_in([
:load,
destination_path,
:data
])
|> get_in(destination_path)
|> case do
%page{results: results} when page in [Ash.Page.Keyset, Ash.Page.Offset] ->
results
@ -998,34 +1400,14 @@ defmodule Ash.Actions.Load do
path,
relationship,
source_query,
request_path,
request_opts,
lazy?
lazy?,
source_data_path,
join_request_path
) do
{offset, limit} = offset_and_limit(base_query)
source_data =
case path do
[] ->
get_in(data, request_path ++ [:data, :results])
path ->
data =
data
|> get_in(request_path)
|> Kernel.||(%{})
|> Map.get(:load, %{})
|> Map.get(Enum.reverse(Enum.map(path, & &1.name)), %{})
|> Map.get(:data, %{})
if is_map(data) do
data
|> Map.values()
|> Enum.flat_map(&List.wrap/1)
else
data
end
end
source_data = get_in(data, source_data_path)
lazy_load_or(
source_data,
@ -1086,9 +1468,9 @@ defmodule Ash.Actions.Load do
relationship,
request_opts,
source_data,
request_path,
path,
data
data,
join_request_path
)
true ->
@ -1097,7 +1479,10 @@ defmodule Ash.Actions.Load do
|> Ash.Query.do_filter(relationship.filter)
|> Ash.Query.sort(relationship.sort, prepend?: true)
|> remove_relationships_from_load()
|> read(relationship.read_action, request_opts)
|> read(
relationship.read_action,
request_opts
)
end
end
)
@ -1118,9 +1503,9 @@ defmodule Ash.Actions.Load do
relationship,
request_opts,
_source_data,
request_path,
path,
data
_path,
data,
join_request_path
) do
query
|> Ash.Query.set_context(relationship.context)
@ -1133,14 +1518,8 @@ defmodule Ash.Actions.Load do
new_results =
if relationship.type == :many_to_many do
results = Enum.with_index(results)
join_path = path ++ [relationship.join_relationship]
join_data =
data
|> get_in(request_path ++ [:load])
|> Kernel.||(%{})
|> Map.get(join_path, %{})
|> Map.get(:data, [])
join_data = get_in(data, join_request_path ++ [:data])
destination_attribute_on_join_resource_type =
Ash.Resource.Info.attribute(
@ -1258,39 +1637,22 @@ defmodule Ash.Actions.Load do
end
end
defp true_load_query(relationship, query, data, path, request_path) do
{source_attribute, path} =
if relationship.type == :many_to_many do
join_relationship = join_relationship(relationship)
{relationship.destination_attribute_on_join_resource,
join_relationship_path(path, join_relationship) |> Enum.map(& &1.name)}
defp true_load_query(relationship, query, data, source_data_path, join_relationship_path) do
{source_attribute, source_data_path} =
if relationship.type == :many_to_many && !lateral_join?(query, relationship, :unknown) do
{relationship.destination_attribute_on_join_resource, join_relationship_path ++ [:data]}
else
{relationship.source_attribute, path |> Enum.reverse() |> Enum.map(& &1.name)}
{relationship.source_attribute, source_data_path}
end
source_data =
case path do
[] ->
get_in(data, request_path ++ [:data, :results])
path ->
data
|> get_in(request_path)
|> Kernel.||(%{})
|> Map.get(:load, %{})
|> Map.get(path, %{})
|> Map.get(:data, %{})
end
case source_data do
case get_in(data, source_data_path) do
%{data: empty} when empty in [[], nil] ->
:nothing
empty when empty in [[], nil] ->
:nothing
_ ->
source_data ->
get_query(query, relationship, source_data, source_attribute)
end
end
@ -1323,7 +1685,8 @@ defmodule Ash.Actions.Load do
end
ids =
Enum.flat_map(related_data, fn
related_data
|> Enum.flat_map(fn
{_, data} when is_list(data) ->
Enum.map(data, &Map.get(&1, source_attribute))

895
lib/ash/actions/read.ex
View file

File diff suppressed because it is too large Load diff

25
lib/ash/engine/engine.ex
View file

@ -367,8 +367,21 @@ defmodule Ash.Engine do
end
end
defp depends_on_summary(request, state) do
dependencies = state.dependencies[request.path] || []
defp depends_on_summary(%{path: request_path} = request, state) do
dependencies =
state.dependencies_waiting_on_request
|> Kernel.||([])
|> Enum.filter(fn
{^request_path, _} ->
true
_ ->
false
end)
|> Enum.map(fn {_, dep} ->
{:lists.droplast(dep), List.last(dep)}
end)
|> Enum.concat(state.dependencies[request_path] || [])
if Enum.empty?(dependencies) do
" state: #{request.state}"
@ -378,7 +391,13 @@ defmodule Ash.Engine do
end
defp name_of({path, dep}, state) do
"#{inspect(Enum.find(state.requests, &(&1.path == path)).name)}.#{dep}"
case Enum.find(state.requests, &(&1.path == path)) do
nil ->
"unknown dependency: #{inspect(path, structs: false)} -> #{inspect(dep)}"
request ->
"#{request.name} -> #{inspect(dep)}"
end
end
defp run_iteration(

8
lib/ash/engine/request.ex
View file

@ -365,13 +365,13 @@ defmodule Ash.Engine.Request do
end
end
def summarize(%{name: name, action: %{name: action}, resource: resource})
def summarize(%{path: path, name: name, action: %{name: action}, resource: resource})
when not is_nil(resource) do
"#{name}: #{inspect(resource)}.#{action}"
"#{inspect(path, structs: false)}\n#{name}: #{inspect(resource)}.#{action}"
end
def summarize(%{name: name}) do
name
def summarize(%{path: path, name: name}) do
"#{inspect(path, structs: false)} - #{name}"
end
def sort_and_clean_notifications(notifications) do

2
lib/ash/error/exception.ex
View file

@ -16,7 +16,7 @@ defmodule Ash.Error.Exception do
error_context: [],
vars: [],
path: [],
stacktrace: [],
stacktrace: nil,
class: unquote(opts)[:class]
]

36
lib/ash/filter/runtime.ex
View file

@ -44,10 +44,14 @@ defmodule Ash.Filter.Runtime do
refs_to_load =
refs_to_load
|> Enum.map(fn
%{attribute: %Ash.Resource.Calculation{load: nil} = calc} ->
{calc.name, calc}
|> Enum.reject(fn
%{attribute: %Ash.Resource.Calculation{}} ->
true
_ ->
false
end)
|> Enum.map(fn
%{attribute: %{name: name}} ->
name
end)
@ -537,10 +541,9 @@ defmodule Ash.Filter.Runtime do
attribute: %Ash.Query.Calculation{
module: module,
opts: opts,
context: context,
name: name
context: context
}
} = ref,
},
record,
parent,
resource
@ -573,13 +576,20 @@ defmodule Ash.Filter.Runtime do
|> resolve_expr(record, parent, resource)
end
else
# This is problematic with variadic loads
resolve_ref(
%{ref | attribute: %Ash.Resource.Attribute{name: name}},
record,
parent,
resource
)
# We need to rewrite this
# As it stands now, it will evaluate the calculation
# once per expanded result. I'm not sure what that will
# look like though.
case module.calculate([record], opts, context) do
[result] ->
{:ok, result}
{:ok, [result]} ->
{:ok, result}
_ ->
{:ok, nil}
end
end
end

2
lib/ash/query/calculation.ex
View file

@ -63,6 +63,8 @@ defmodule Ash.Query.Calculation do
:initial_limit,
:initial_offset,
:context,
:tenant,
:tracer,
:ash
])

252
lib/ash/query/query.ex
View file

@ -863,7 +863,6 @@ defmodule Ash.Query do
# Loading relationships with a query
Ash.Query.load(query, [comments: [author: author_query]])
```
"""
@spec load(
t() | Ash.Resource.t(),
@ -905,29 +904,8 @@ defmodule Ash.Query do
resource_calculation.filterable?,
resource_calculation.load
) do
fields_to_select =
resource_calculation.select
|> Kernel.||([])
|> Enum.concat(module.select(query, opts, calculation.context) || [])
|> Enum.uniq()
|> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
loads =
module.load(
query,
opts,
Map.put(calculation.context, :context, query.context)
)
|> Ash.Actions.Helpers.validate_calculation_load!(module)
|> Enum.concat(List.wrap(resource_calculation.load))
|> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
calculation = %{
calculation
| load: field,
select: fields_to_select,
required_loads: loads
}
calculation =
select_and_load_calc(resource_calculation, %{calculation | load: field}, query)
Map.update!(query, :calculations, &Map.put(&1, field, calculation))
end
@ -941,34 +919,132 @@ defmodule Ash.Query do
end)
end
defp do_load(query, field) do
cond do
match?(%Ash.Query.Calculation{}, field) ->
calculation = field
@doc false
def select_and_load_calc(resource_calculation, calculation, query) do
module = calculation.module
opts = calculation.opts
resource_calculation_select =
if resource_calculation do
List.wrap(resource_calculation.select)
else
[]
end
resource_calculation_load =
if resource_calculation do
List.wrap(resource_calculation.load)
else
[]
end
fields_to_select =
calculation.module.select(query, calculation.opts, calculation.context)
resource_calculation_select
|> Kernel.||([])
|> Enum.concat(module.select(query, opts, calculation.context) || [])
|> Enum.uniq()
|> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
loads =
calculation.module.load(
module.load(
query,
calculation.opts,
opts,
Map.put(calculation.context, :context, query.context)
)
|> Ash.Actions.Helpers.validate_calculation_load!(calculation.module)
|> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
|> Ash.Actions.Helpers.validate_calculation_load!(module)
|> Enum.concat(resource_calculation_load)
|> reify_calculations(query)
calculation = %{
calculation
| load: field,
required_loads: loads,
select: fields_to_select
}
%{calculation | select: fields_to_select, required_loads: loads}
end
Map.update!(query, :calculations, &Map.put(&1, calculation.name, calculation))
@doc false
def reify_calculations(loads, query) do
loads
|> List.wrap()
|> Enum.map(fn
{load, args} ->
if resource_calculation = Ash.Resource.Info.calculation(query.resource, load) do
{module, opts} = resource_calculation.calculation
with {:ok, args} <- validate_calculation_arguments(resource_calculation, args),
{:ok, calculation} <-
Calculation.new(
resource_calculation.name,
module,
opts,
{resource_calculation.type, resource_calculation.constraints},
Map.put(args, :context, query.context),
resource_calculation.filterable?,
resource_calculation.load
) do
select_and_load_calc(resource_calculation, %{calculation | load: load}, query)
else
_ ->
{load, args}
end
else
if relationship = Ash.Resource.Info.relationship(query.resource, load) do
related_query = new(relationship.destination)
{load, reify_calculations(args, related_query)}
else
{load, args}
end
end
load ->
if resource_calculation = Ash.Resource.Info.calculation(query.resource, load) do
case resource_calc_to_calc(query, load, resource_calculation) do
{:error, _} ->
load
{:ok, calc} ->
calc
end
else
if relationship = Ash.Resource.Info.relationship(query.resource, load) do
{load, relationship.destination |> new() |> set_tenant(query.tenant)}
else
load
end
end
end)
|> case do
[%Ash.Query{} = query] -> query
other -> other
end
end
@doc false
def resource_calc_to_calc(query, name, resource_calculation) do
with %{calculation: {module, opts}} <- resource_calculation,
{:ok, args} <- validate_calculation_arguments(resource_calculation, %{}),
{:ok, calculation} <-
Calculation.new(
resource_calculation.name,
module,
opts,
{resource_calculation.type, resource_calculation.constraints},
Map.put(args, :context, query.context),
resource_calculation.filterable?,
resource_calculation.load
) do
{:ok, select_and_load_calc(resource_calculation, %{calculation | load: name}, query)}
end
end
defp do_load(query, field) do
cond do
match?(%Ash.Query.Calculation{}, field) ->
Map.update!(
query,
:calculations,
&Map.put(
&1,
field.name,
select_and_load_calc(nil, field, query)
)
)
Ash.Resource.Info.attribute(query.resource, field) ->
Ash.Query.ensure_selected(query, field)
@ -1038,47 +1114,12 @@ defmodule Ash.Query do
end
resource_calculation = Ash.Resource.Info.calculation(query.resource, field) ->
{module, opts} = resource_calculation.calculation
with {:ok, args} <- validate_calculation_arguments(resource_calculation, %{}),
{:ok, calculation} <-
Calculation.new(
resource_calculation.name,
module,
opts,
{resource_calculation.type, resource_calculation.constraints},
Map.put(args, :context, query.context),
resource_calculation.filterable?,
resource_calculation.load
) do
fields_to_select =
resource_calculation.select
|> Kernel.||([])
|> Enum.concat(module.select(query, opts, calculation.context) || [])
|> Enum.uniq()
|> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
loads =
module.load(
query,
opts,
Map.put(calculation.context, :context, query.context)
)
|> Ash.Actions.Helpers.validate_calculation_load!(module)
|> Enum.concat(List.wrap(resource_calculation.load))
|> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
calculation = %{
calculation
| load: field,
required_loads: loads,
select: fields_to_select
}
Map.update!(query, :calculations, &Map.put(&1, field, calculation))
else
case resource_calc_to_calc(query, resource_calculation.name, resource_calculation) do
{:error, error} ->
add_error(query, :load, error)
{:ok, calc} ->
Map.update!(query, :calculations, &Map.put(&1, field, calc))
end
true ->
@ -1705,7 +1746,7 @@ defmodule Ash.Query do
)
|> Ash.Actions.Helpers.validate_calculation_load!(module)
|> Enum.concat(List.wrap(calculation.required_loads))
|> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
|> reify_calculations(query)
calculation = %{calculation | select: fields_to_select, required_loads: loads}
%{query | calculations: Map.put(query.calculations, name, calculation)}
@ -1735,30 +1776,12 @@ defmodule Ash.Query do
resource_calculation.filterable?,
resource_calculation.load
) do
fields_to_select =
resource_calculation.select
|> Kernel.||([])
|> Enum.concat(module.select(query, opts, calculation.context) || [])
|> Enum.uniq()
|> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
loads =
module.load(
query,
opts,
Map.put(calculation.context, :context, query.context)
calculation =
select_and_load_calc(
resource_calculation,
%{calculation | load: nil, name: as_name},
query
)
|> Ash.Actions.Helpers.validate_calculation_load!(module)
|> Enum.concat(List.wrap(resource_calculation.load))
|> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
calculation = %{
calculation
| name: as_name,
load: nil,
select: fields_to_select,
required_loads: loads
}
Map.update!(query, :calculations, &Map.put(&1, as_name, calculation))
else
@ -2103,24 +2126,31 @@ defmodule Ash.Query do
keys
|> Enum.reduce(query, fn key, query ->
if key in [:api, :resource] do
query
else
struct(query, [{key, Map.get(new, key)}])
end
do_unset(query, key, new)
end)
end
def unset(query, key) do
if key in [:api, :resource] do
to_query(query)
else
new = new(query.resource)
do_unset(query, key, new)
query
|> to_query()
|> struct([{key, Map.get(new, key)}])
end
defp do_unset(query, key, _new) when key in [:api, :resource] do
query
end
defp do_unset(query, :load, new) do
query = unset(query, [:calculations, :aggregates])
struct(query, [{:load, Map.get(new, :load)}])
end
defp do_unset(query, key, new) do
struct(query, [{key, Map.get(new, key)}])
end
@doc "Return the underlying data layer query for an ash query"

1
lib/ash/resource/manual_relationship/manual_relationship.ex
View file

@ -6,7 +6,6 @@ defmodule Ash.Resource.ManualRelationship do
@type context :: %{
relationship: Ash.Resource.Relationships.relationship(),
query: Ash.Query.t(),
root_query: Ash.Query.t(),
actor: term,
tenant: term,
authorize?: term,

2
lib/ash/type/function.ex
View file

@ -4,7 +4,7 @@ defmodule Ash.Type.Function do
If the type would be dumped to a native format, `:erlang.term_to_binary(term, [:safe])` is used.
Please keep in mind, this is not safe to use with external input. This could easily cause you t
Please keep in mind, this is *NOT SAFE* to use with external input.
More information available here: https://erlang.org/doc/man/erlang.html#binary_to_term-2
"""

129
read-action-refactor.md Normal file
View file

@ -0,0 +1,129 @@
# Read Actions & Data Loading Refactor
Problems:
1. Loading data - lots of expensive comparison operations being performed to check what's been loaded, etc.
- calculation metadata is recursively populated
- no simple format for declaring the data that needs to be loaded
2. Read action flow
Idea: use Ash.Flow to implement read (and in future all) actions.
Needs:
1. First class ability of steps to generate new steps.
2. (Maybe) ergonomics improvements to some conditional steps, eg: `branch` rather than `if/else`.
3. Maybe conditional dependencies?
# Actions:
1. Feature flag for switching between `read.ex` and `read_flow.ex`.
2. Iterate until all the tests pass.
# Suspicions:
1. Ash.Flow step generator.
2. Ash.Flow if/else steps.
3. Conditional dependencies?
This is too big to fail, so: let's just do calculations.
New structure for calculations:
Benefits:
simpler to manipulate and check what calculations have been loaded, and comparing what calculations have been loaded no longer *also must* compare their definitions.
Note:
we end up removing `name` and `load` from `Ash.Query.Calculation` (maybe eventually? In Ash 3.0? If we can achieve backwards compatibility then we'd leave them. Maybe won't matter, needs to be thought about)
Filters will be fine without the keys above because they don't need to call them anything to put them into an expression
```elixir
@type calculation_definition :: %{name: atom(), target: target()}
@type target :: {:calculations, atom()} | {:top_level, atom()}
# better names for this key to be workshopped
calculations_to_load: %{
score1: %{args: %{}, load_as: :}
full_name: %{arg1: 10},
full_name: %{arg1: 11},
full_name: %{arg1: 12},
some_random_shit: %{arg1: 10}
},
calculation_definitions: %{
# Supports anonymous calculations & resource calculations
%{name: :full_name, target: {:top_level, :full_name}} => %Ash.Query.Calculation{},
%{name: :full_name, target: {:calculations, :full_name_2}} => %Ash.Query.Calculation{}
}
```
Scratch
```elixir
%{
full_name: %Calculation{},
some_random_shit: %Calculation{name: :some_random_shit}
}
%Calculation{name: :full_name, load: , context: %{arg1: 10, actor: ..., ...., ...}}
calculations do
calculate :full_name, :string,
expr(first_name <> ^arg(:separator) <> last_name) do
argument :separator, :string do
allow_nil? false
default " "
end
end
end
User
|> Ash.Query.load(full_name: %{separator: "~"})
|> Ash.Query.calculate(:thing, Ash.Query.Calculation.new(...))
%Resource{
thing: thing,
calculations: %{
thing2: thing
}
}
%{name: :thing, target: :thing}
%{name: :thing, target: {:calculations, :thing2}}
calculations: %{
full_name: %{args: %{separator: "~"}}
},
calculation_definitions: %{
%{name: :full_name, load: :full_name} => %Ash.Query.Calculation{}
}
%Resource{
name: :value,
calculations: %{}
}
Ash.Resource.Info.calculation(query.resource, calc_name,)
some_read_action
|> read_them()
|> Enum.map(...)
|> calculation(....)
query
|> Ash.Query.load_calculation_as(:score, :score_1, %{arg: 1})
|> Ash.Query.load_calculation_as(:score, :score_1, %{arg: 1})
```
```elixir
# What calculate looks like now, context jumbled in with context
def calculate(records, opts, %{arg1: arg1, actor: actor}) do
end
# What it should look like < 3.0 >
def calculate(records, opts, args, %CalculationContext{actor: actor, tenant: tenant, authorize?: authorize?} = context) do
end
```

2
test/actions/async_load_test.exs
View file

@ -219,8 +219,6 @@ defmodule Ash.Test.Actions.AsyncLoadTest do
|> manage_relationship(:author, author, type: :append_and_remove)
|> Api.create!()
Application.put_env(:foo, :bar, true)
authorized_posts =
author
|> Api.load!(:authorized_actor_posts)

213
test/calculation_test.exs
View file

@ -4,6 +4,31 @@ defmodule Ash.Test.CalculationTest do
require Ash.Query
defmodule FriendLink do
use Ash.Resource,
data_layer: Ash.DataLayer.Ets
ets do
private? true
end
actions do
defaults [:create, :read, :update, :destroy]
end
relationships do
belongs_to :source, Ash.Test.CalculationTest.Friend do
allow_nil? false
primary_key? true
end
belongs_to :target, Ash.Test.CalculationTest.Friend do
allow_nil? false
primary_key? true
end
end
end
defmodule Concat do
# An example concatenation calculation, that accepts the delimiter as an argument
use Ash.Calculation
@ -89,9 +114,9 @@ defmodule Ash.Test.CalculationTest do
def load(_query, _opts, args) do
if args[:only_special] do
query =
__MODULE__.User
Ash.Test.CalculationTest.User
|> Ash.Query.filter(special == true)
|> Ash.Query.ensure_selected(:full_name)
|> Ash.Query.load(:full_name)
[best_friends_of_me: query]
else
@ -123,6 +148,32 @@ defmodule Ash.Test.CalculationTest do
end
end
defmodule FriendsNames do
use Ash.Calculation
def load(_query, _opts, _) do
[
friends:
Ash.Test.CalculationTest.User
|> Ash.Query.load([:full_name, :prefix])
|> Ash.Query.limit(5)
]
end
def calculate(records, _opts, _) do
Enum.map(records, fn record ->
record.friends
|> Enum.map_join(" | ", fn friend ->
if friend.prefix do
friend.prefix <> " " <> friend.full_name
else
friend.full_name
end
end)
end)
end
end
defmodule User do
@moduledoc false
use Ash.Resource, data_layer: Ash.DataLayer.Ets
@ -146,6 +197,7 @@ defmodule Ash.Test.CalculationTest do
uuid_primary_key :id
attribute :first_name, :string
attribute :last_name, :string
attribute :prefix, :string
attribute :special, :boolean
end
@ -171,7 +223,8 @@ defmodule Ash.Test.CalculationTest do
:string,
{ConcatWithLoad, keys: [:full_name, :full_name_plus_full_name]}
calculate :slug, :string, expr(full_name <> "123"), load: [:full_name]
calculate :slug, :string, expr(full_name <> "123")
calculate :friends_names, :string, FriendsNames
calculate :expr_full_name, :string, expr(first_name <> " " <> last_name)
@ -181,7 +234,10 @@ defmodule Ash.Test.CalculationTest do
calculate :best_friends_name, :string, BestFriendsName
calculate :names_of_best_friends_of_me, :string, NamesOfBestFriendsOfMe
calculate :names_of_best_friends_of_me, :string, NamesOfBestFriendsOfMe do
argument :only_special, :boolean, default: false
end
calculate :name_with_users_name, :string, NameWithUsersName
calculate :full_name_with_salutation,
@ -237,6 +293,12 @@ defmodule Ash.Test.CalculationTest do
has_many :best_friends_of_me, __MODULE__ do
destination_attribute :best_friend_id
end
many_to_many :friends, __MODULE__ do
through FriendLink
destination_attribute_on_join_resource :target_id
source_attribute_on_join_resource :source_id
end
end
end
@ -245,7 +307,8 @@ defmodule Ash.Test.CalculationTest do
use Ash.Registry
entries do
entry(User)
entry User
entry FriendLink
end
end
@ -268,6 +331,7 @@ defmodule Ash.Test.CalculationTest do
User
|> Ash.Changeset.new(%{first_name: "brian", last_name: "cranston"})
|> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
|> Ash.Changeset.manage_relationship(:friends, user1, type: :append_and_remove)
|> Api.create!()
%{user1: user1, user2: user2}
@ -321,26 +385,10 @@ defmodule Ash.Test.CalculationTest do
]
end
test "it doesn't reload anything specified by the load callback if its already been loaded when using `lazy?: true`" do
full_names =
User
|> Ash.Query.load(:full_name_plus_full_name)
|> Api.read!()
|> Enum.map(&%{&1 | full_name: &1.full_name <> " more"})
|> Api.load!(:full_name_plus_full_name, lazy?: true)
|> Enum.map(& &1.full_name_plus_full_name)
|> Enum.sort()
assert full_names == [
"brian cranston more brian cranston more",
"zach daniel more zach daniel more"
]
end
test "it reloads anything specified by the load callback if its already been loaded when using `lazy?: false`" do
full_names =
User
|> Ash.Query.load(:full_name_plus_full_name)
|> Ash.Query.load([:full_name, :full_name_plus_full_name])
|> Api.read!()
|> Enum.map(&%{&1 | full_name: &1.full_name <> " more"})
|> Api.load!(:full_name_plus_full_name)
@ -515,34 +563,97 @@ defmodule Ash.Test.CalculationTest do
assert full_names == ["bob", "brian cranston", "zach daniel"]
end
# test "loading calculations with different relationship dependencies won't collide", %{
# user1: %{id: user1_id} = user1
# } do
# user3 =
# User
# |> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
# |> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
# |> Api.create!()
test "loading calculations with different relationship dependencies won't collide", %{
user1: %{id: user1_id} = user1
} do
User
|> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
|> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
|> Api.create!()
# assert %{
# calculations: %{
# names_of_best_friends_of_me: "brian cranston - chidi anagonye",
# names_of_special_best_friends_of_me: "chidi anagonye"
# }
# } =
# User
# |> Ash.Query.filter(id == ^user1_id)
# |> Ash.Query.load_calculation_as(
# :names_of_best_friends_of_me,
# :names_of_special_best_friends_of_me,
# %{
# special: true
# }
# )
# |> Ash.Query.load_calculation_as(
# :names_of_best_friends_of_me,
# :names_of_best_friends_of_me
# )
# |> Api.read_one!()
# end
assert %{
calculations: %{
names_of_best_friends_of_me: "brian cranston - chidi anagonye",
names_of_special_best_friends_of_me: "chidi anagonye"
}
} =
User
|> Ash.Query.filter(id == ^user1_id)
|> Ash.Query.load_calculation_as(
:names_of_best_friends_of_me,
:names_of_special_best_friends_of_me,
%{
only_special: true
}
)
|> Ash.Query.load_calculation_as(
:names_of_best_friends_of_me,
:names_of_best_friends_of_me
)
|> Api.read_one!()
end
test "loading calculations with different relationship dependencies won't collide, when manually loading one of the deps",
%{
user1: %{id: user1_id} = user1
} do
User
|> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
|> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
|> Api.create!()
assert %{
calculations: %{
names_of_best_friends_of_me: "brian cranston - chidi anagonye",
names_of_special_best_friends_of_me: "chidi anagonye"
}
} =
User
|> Ash.Query.filter(id == ^user1_id)
|> Ash.Query.load(:best_friends_of_me)
|> Ash.Query.load_calculation_as(
:names_of_best_friends_of_me,
:names_of_special_best_friends_of_me,
%{
only_special: true
}
)
|> Ash.Query.load_calculation_as(
:names_of_best_friends_of_me,
:names_of_best_friends_of_me
)
|> Api.read_one!()
end
test "calculations that depend on many to many relationships will load", %{user2: user2} do
assert %{
friends_names: "zach daniel"
} =
User
|> Ash.Query.filter(id == ^user2.id)
|> Ash.Query.load(:friends_names)
|> Api.read_one!()
end
test "when already loading a calculation's dependency it is used" do
full_names =
User
|> Ash.Query.load([:full_name, :full_name_plus_full_name])
|> Api.read!()
|> Enum.map(& &1.full_name_plus_full_name)
|> Enum.sort()
assert full_names == ["brian cranston brian cranston", "zach daniel zach daniel"]
end
test "when already loading a nested calculation dependency, it is used" do
best_friends_names =
User
|> Ash.Query.load([:best_friends_name, best_friend: [:full_name]])
|> Api.read!()
|> Enum.map(& &1.best_friends_name)
|> Enum.sort()
assert best_friends_names == [nil, "zach daniel"]
end
end