improvement: rewrite calculation loader

Calculation loading is complex because different calculations can
depend on differently parameterized things. FOr example:

```elixir
def load(_, _, _), do: [foo: %{arg: 1}]
def load(_, _, _), do: [foo: %{arg: 2}]
```

The previous naive implementation would simply merge all of the calculation loads, which naturally would not work. Now we ensure that we load each requirement in isolation.

lib/ash/actions/helpers.ex

@@ -152,7 +152,7 @@ defmodule Ash.Actions.Helpers do
           if Keyword.has_key?(opts, :actor) do
             Keyword.put_new(opts, :authorize?, true)
           else
-            opts
+            Keyword.put(opts, :authorize?, opts[:authorize?] || Keyword.has_key?(opts, :actor))
           end
       end
     else
@@ -202,7 +202,7 @@ defmodule Ash.Actions.Helpers do
         :ok
 
       missed ->
-        case Application.get_env(:ash, :missed_notifications, :ignore) do
+        case Application.get_env(:ash, :missed_notifications, :warn) do
           :ignore ->
             :ok
 

lib/ash/actions/load.ex

@@ -407,23 +407,9 @@ defmodule Ash.Actions.Load do
   end
 
   defp do_requests(relationship, lazy?, opts, request_path, related_query, path, root_query) do
-    load_request =
-      load_request(
-        relationship,
-        lazy?,
-        opts,
-        request_path,
-        related_query,
-        root_query,
-        path
-      )
-
-    case relationship.type do
-      :many_to_many ->
-        if lateral_join?(related_query, relationship, :unknown) do
-          [load_request]
-        else
     join_assoc_request =
+      if relationship.type == :many_to_many &&
+           !lateral_join?(related_query, relationship, :unknown) do
         join_assoc_request(
           relationship,
           request_path,
@@ -433,15 +419,156 @@ defmodule Ash.Actions.Load do
           opts,
           lazy?
         )
-
-          [join_assoc_request, load_request]
       end
 
-      _ ->
+    join_request_path =
+      if join_assoc_request do
+        join_assoc_request.path
+      end
+
+    load_request =
+      load_request(
+        relationship,
+        lazy?,
+        opts,
+        request_path,
+        related_query,
+        root_query,
+        path,
+        join_request_path
+      )
+
+    if join_assoc_request do
+      [join_assoc_request, load_request]
+    else
       [load_request]
     end
   end
 
+  @doc false
+  def calc_dep_requests(
+        relationship,
+        lazy?,
+        opts,
+        calc_dep,
+        path,
+        root_query,
+        select,
+        load
+      ) do
+    join_assoc_request =
+      if relationship.type == :many_to_many &&
+           !lateral_join?(calc_dep.query, relationship, :unknown) do
+        calc_dep_join_assoc_request(
+          relationship,
+          path,
+          root_query,
+          path,
+          opts,
+          lazy?,
+          calc_dep
+        )
+      end
+
+    load_request =
+      calc_dep_load_request(
+        relationship,
+        lazy?,
+        opts,
+        calc_dep,
+        root_query,
+        path,
+        select,
+        load
+      )
+
+    if join_assoc_request do
+      [join_assoc_request, load_request]
+    else
+      [load_request]
+    end
+  end
+
+  defp calc_dep_load_request(
+         relationship,
+         lazy?,
+         opts,
+         calc_dep,
+         root_query,
+         path,
+         select,
+         load
+       ) do
+    {parent_dep_path, parent_data_path, parent_query_path} =
+      case calc_dep.path do
+        [] ->
+          {path ++ [:fetch, :data], path ++ [:fetch, :data, :results], path ++ [:fetch, :query]}
+
+        dependent_path ->
+          {relationship, query} = List.last(dependent_path)
+
+          depended_dep = %{
+            type: :relationship,
+            path: :lists.droplast(dependent_path),
+            query: query,
+            relationship: relationship
+          }
+
+          data = path ++ [:calc_dep, depended_dep, :data]
+
+          {data, data, path ++ [:calc_dep, depended_dep, :query]}
+      end
+
+    this_request_path = path ++ [:calc_dep, calc_dep]
+
+    dependencies = [
+      this_request_path ++ [:authorization_filter],
+      parent_dep_path,
+      parent_query_path
+    ]
+
+    dependencies =
+      if relationship.type == :many_to_many &&
+           !lateral_join?(calc_dep.query, relationship, :unknown) do
+        dependencies ++ [this_request_path ++ [:join, :data]]
+      else
+        dependencies
+      end
+
+    rel_string =
+      calc_dep.path
+      |> Enum.map(&elem(&1, 0))
+      |> Enum.concat([calc_dep.relationship])
+      |> Enum.join(".")
+
+    actual_query = calc_dep.query |> Ash.Query.select(select) |> Ash.Query.load(load)
+
+    Request.new(
+      action:
+        calc_dep.query.action || Ash.Resource.Info.primary_action(relationship.destination, :read),
+      resource: relationship.destination,
+      name: "calc dep #{rel_string}",
+      api: calc_dep.query.api,
+      path: this_request_path,
+      query: actual_query,
+      data:
+        calc_dep_data(
+          relationship,
+          lazy?,
+          dependencies,
+          this_request_path,
+          calc_dep,
+          path,
+          root_query,
+          opts,
+          parent_data_path,
+          parent_query_path,
+          this_request_path ++ [:join],
+          actual_query
+        )
+    )
+  end
+
   defp load_request(
          relationship,
          lazy?,
@@ -449,7 +576,8 @@ defmodule Ash.Actions.Load do
          request_path,
          related_query,
          root_query,
-         path
+         path,
+         join_request_path
        ) do
     relationship_path = Enum.reverse(Enum.map([relationship | path], &Map.get(&1, :name)))
 
@@ -521,11 +649,149 @@ defmodule Ash.Actions.Load do
           related_query,
           path,
           root_query,
-          opts
+          opts,
+          join_request_path
         )
     )
   end
 
+  defp calc_dep_data(
+         %{manual: manual} = relationship,
+         lazy?,
+         dependencies,
+         _this_request_path,
+         calc_dep,
+         _path,
+         root_query,
+         request_opts,
+         parent_data_path,
+         _parent_query_path,
+         _join_request_path,
+         actual_query
+       )
+       when not is_nil(manual) do
+    {mod, opts} =
+      case manual do
+        {mod, opts} ->
+          {mod, opts}
+
+        mod ->
+          {mod, []}
+      end
+
+    Request.resolve(dependencies, fn data ->
+      data = get_in(data, parent_data_path)
+
+      lazy_load_or(
+        data,
+        lazy?,
+        relationship.name,
+        root_query.query.api,
+        actual_query,
+        request_opts,
+        fn ->
+          data
+          |> mod.load(opts, %{
+            relationship: relationship,
+            query: actual_query,
+            actor: request_opts[:actor],
+            authorize?: request_opts[:authorize?],
+            api: calc_dep.query.api,
+            tenant: root_query.tenant
+          })
+          |> case do
+            {:ok, result} ->
+              # TODO: this will result in quite a few requests potentially for aggs/calcs
+              # This should be optimized.
+              Enum.reduce_while(result, {:ok, %{}}, fn {key, records}, {:ok, acc} ->
+                case calc_dep.query.api.load(records, %{actual_query | load: []},
+                       lazy?: true,
+                       tenant: calc_dep.query.tenant,
+                       actor: request_opts[:actor],
+                       authorize?: request_opts[:authorize?],
+                       tracer: request_opts[:tracer]
+                     ) do
+                  {:ok, results} ->
+                    {:cont, {:ok, Map.put(acc, key, results)}}
+
+                  {:error, error} ->
+                    {:halt, {:error, error}}
+                end
+              end)
+
+            {:error, error} ->
+              {:error, error}
+          end
+        end
+      )
+    end)
+  end
+
+  defp calc_dep_data(
+         relationship,
+         lazy?,
+         dependencies,
+         this_request_path,
+         calc_dep,
+         path,
+         _root_query,
+         opts,
+         parent_data_path,
+         parent_query_path,
+         join_request_path,
+         actual_query
+       ) do
+    Request.resolve(dependencies, fn data ->
+      base_query =
+        case get_in(data, this_request_path ++ [:authorization_filter]) do
+          nil ->
+            actual_query
+
+          authorization_filter ->
+            Ash.Query.filter(actual_query, ^authorization_filter)
+        end
+
+      source_query = get_in(data, parent_query_path)
+
+      source_query =
+        if calc_dep.query.tenant do
+          Ash.Query.set_tenant(source_query, calc_dep.query.tenant)
+        else
+          source_query
+        end
+
+      with {:ok, new_query} <-
+             true_load_query(
+               relationship,
+               base_query,
+               data,
+               parent_data_path,
+               join_request_path
+             ),
+           {:ok, results} <-
+             run_actual_query(
+               new_query,
+               base_query,
+               data,
+               path,
+               relationship,
+               source_query,
+               opts,
+               lazy?,
+               parent_data_path,
+               join_request_path
+             ) do
+        {:ok, results}
+      else
+        :nothing ->
+          {:ok, []}
+
+        {:error, error} ->
+          {:error, error}
+      end
+    end)
+  end
+
   defp data(
          %{manual: manual} = relationship,
          lazy?,
@@ -535,7 +801,8 @@ defmodule Ash.Actions.Load do
          related_query,
          path,
          root_query,
-         request_opts
+         request_opts,
+         _join_request_path
        )
        when not is_nil(manual) do
     {mod, opts} =
@@ -626,7 +893,8 @@ defmodule Ash.Actions.Load do
          related_query,
          path,
          root_query,
-         opts
+         opts,
+         join_request_path
        ) do
     Request.resolve(dependencies, fn data ->
       base_query =
@@ -662,13 +930,31 @@ defmodule Ash.Actions.Load do
           source_query
         end
 
+      data_path =
+        if relationship.type == :many_to_many do
+          join_relationship = join_relationship(relationship)
+
+          join_relationship_path(path, join_relationship) |> Enum.map(& &1.name)
+        else
+          path |> Enum.reverse() |> Enum.map(& &1.name)
+        end
+
+      source_data_path =
+        case path do
+          [] ->
+            request_path ++ [:data, :results]
+
+          _path ->
+            request_path ++ [:load] ++ [data_path] ++ [:data]
+        end
+
       with {:ok, new_query} <-
              true_load_query(
                relationship,
                base_query,
                data,
-               path,
+               source_data_path,
-               request_path
+               join_request_path
              ),
            {:ok, results} <-
              run_actual_query(
@@ -678,9 +964,10 @@ defmodule Ash.Actions.Load do
                path,
                relationship,
                source_query,
-               request_path,
                opts,
-               lazy?
+               lazy?,
+               source_data_path,
+               join_request_path
              ) do
         {:ok, results}
       else
@@ -757,22 +1044,6 @@ defmodule Ash.Actions.Load do
         ]
       end
 
-    lateral_join? = lateral_join?(related_query, relationship, :unknown)
-
-    dependencies =
-      if lateral_join? do
-        dependencies ++
-          [
-            [
-              :load,
-              Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name],
-              :data
-            ]
-          ]
-      else
-        dependencies
-      end
-
     dependencies = [[:data] | dependencies]
 
     related_query =
@@ -858,13 +1129,24 @@ defmodule Ash.Actions.Load do
                 source_query
               end
 
+            data_path = path |> Enum.reverse() |> Enum.map(& &1.name)
+
+            source_data_path =
+              case path do
+                [] ->
+                  request_path ++ [:data, :results]
+
+                _path ->
+                  request_path ++ [:load] ++ [data_path] ++ [:data]
+              end
+
             with {:ok, new_query} <-
                    true_load_query(
                      join_relationship,
                      base_query,
                      data,
-                     path,
+                     source_data_path,
-                     request_path
+                     nil
                    ),
                  new_query <-
                    add_join_destination_filter(
@@ -872,7 +1154,11 @@ defmodule Ash.Actions.Load do
                      lateral_join?,
                      data,
                      relationship,
-                     Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name]
+                     [
+                       :load,
+                       Enum.reverse(Enum.map(path, &Map.get(&1, :name))) ++ [relationship.name],
+                       :data
+                     ]
                    ),
                  {:ok, results} <-
                    run_actual_query(
@@ -882,9 +1168,129 @@ defmodule Ash.Actions.Load do
                      path,
                      join_relationship,
                      source_query,
-                     request_path,
                      opts,
-                     lazy?
+                     lazy?,
+                     source_data_path,
+                     nil
+                   ) do
+              {:ok, results}
+            else
+              :nothing ->
+                {:ok, []}
+
+              {:error, error} ->
+                {:error, error}
+            end
+        end)
+    )
+  end
+
+  defp calc_dep_join_assoc_request(
+         relationship,
+         path,
+         root_query,
+         path,
+         opts,
+         lazy?,
+         calc_dep
+       ) do
+    {parent_dep_path, parent_data_path, parent_query_path} =
+      case calc_dep.path do
+        [] ->
+          {path ++ [:fetch], path ++ [:fetch, :data, :results], path ++ [:fetch, :query]}
+
+        dependent_path ->
+          {relationship, query} = List.last(dependent_path)
+
+          depended_dep = %{
+            type: :relationship,
+            path: :lists.droplast(dependent_path),
+            query: query,
+            relationship: relationship
+          }
+
+          {path ++ [:calc_dep, depended_dep], path ++ [:calc_dep, depended_dep, :data, :results],
+           path ++ [:calc_dep, depended_dep, :query]}
+      end
+
+    this_request_path = path ++ [:calc_dep, calc_dep]
+    join_relationship = join_relationship(relationship)
+
+    dependencies = [
+      this_request_path ++ [:authorization_filter],
+      parent_dep_path ++ [:data],
+      parent_dep_path ++ [:query]
+    ]
+
+    base_query =
+      join_relationship.destination
+      |> Ash.Query.new(join_relationship.api || root_query.api)
+      |> Ash.Query.set_tenant(calc_dep.query.tenant)
+      |> Ash.Query.set_context(join_relationship.context)
+
+    Request.new(
+      action:
+        calc_dep.query.action ||
+          Ash.Resource.Info.primary_action(join_relationship.destination, :read),
+      resource: relationship.through,
+      name: "load calc_dep join #{join_relationship.name}",
+      api: calc_dep.query.api,
+      path: path ++ [:calc_dep, calc_dep, :join],
+      query: base_query,
+      data:
+        Request.resolve(dependencies, fn
+          data ->
+            base_query =
+              case get_in(
+                     data,
+                     this_request_path ++ [:authorization_filter]
+                   ) do
+                nil ->
+                  base_query
+
+                authorization_filter ->
+                  base_query
+                  |> Ash.Query.do_filter(authorization_filter)
+              end
+
+            source_data = get_in(data, parent_data_path)
+
+            lateral_join? = lateral_join?(calc_dep.query, relationship, source_data)
+
+            source_query =
+              get_in(
+                data,
+                parent_query_path
+              )
+
+            with {:ok, new_query} <-
+                   true_load_query(
+                     join_relationship,
+                     base_query,
+                     data,
+                     parent_data_path,
+                     nil
+                   ),
+                 new_query <-
+                   add_join_destination_filter(
+                     new_query,
+                     lateral_join?,
+                     data,
+                     relationship,
+                     parent_data_path
+                   ),
+                 {:ok, results} <-
+                   run_actual_query(
+                     new_query,
+                     base_query,
+                     data,
+                     path,
+                     relationship,
+                     source_query,
+                     opts,
+                     lazy?,
+                     parent_data_path,
+                     nil
                    ) do
               {:ok, results}
             else
@@ -901,11 +1307,7 @@ defmodule Ash.Actions.Load do
   defp add_join_destination_filter(query, true, data, relationship, destination_path) do
     ids =
       data
-      |> get_in([
+      |> get_in(destination_path)
-        :load,
-        destination_path,
-        :data
-      ])
       |> case do
         %page{results: results} when page in [Ash.Page.Keyset, Ash.Page.Offset] ->
           results
@@ -998,34 +1400,14 @@ defmodule Ash.Actions.Load do
          path,
          relationship,
          source_query,
-         request_path,
          request_opts,
-         lazy?
+         lazy?,
+         source_data_path,
+         join_request_path
        ) do
     {offset, limit} = offset_and_limit(base_query)
 
-    source_data =
+    source_data = get_in(data, source_data_path)
-      case path do
-        [] ->
-          get_in(data, request_path ++ [:data, :results])
-
-        path ->
-          data =
-            data
-            |> get_in(request_path)
-            |> Kernel.||(%{})
-            |> Map.get(:load, %{})
-            |> Map.get(Enum.reverse(Enum.map(path, & &1.name)), %{})
-            |> Map.get(:data, %{})
-
-          if is_map(data) do
-            data
-            |> Map.values()
-            |> Enum.flat_map(&List.wrap/1)
-          else
-            data
-          end
-      end
 
     lazy_load_or(
       source_data,
@@ -1086,9 +1468,9 @@ defmodule Ash.Actions.Load do
               relationship,
               request_opts,
               source_data,
-              request_path,
               path,
-              data
+              data,
+              join_request_path
             )
 
           true ->
@@ -1097,7 +1479,10 @@ defmodule Ash.Actions.Load do
             |> Ash.Query.do_filter(relationship.filter)
             |> Ash.Query.sort(relationship.sort, prepend?: true)
             |> remove_relationships_from_load()
-            |> read(relationship.read_action, request_opts)
+            |> read(
+              relationship.read_action,
+              request_opts
+            )
         end
       end
     )
@@ -1118,9 +1503,9 @@ defmodule Ash.Actions.Load do
          relationship,
          request_opts,
          _source_data,
-         request_path,
+         _path,
-         path,
+         data,
-         data
+         join_request_path
        ) do
     query
     |> Ash.Query.set_context(relationship.context)
@@ -1133,14 +1518,8 @@ defmodule Ash.Actions.Load do
         new_results =
           if relationship.type == :many_to_many do
             results = Enum.with_index(results)
-            join_path = path ++ [relationship.join_relationship]
 
-            join_data =
+            join_data = get_in(data, join_request_path ++ [:data])
-              data
-              |> get_in(request_path ++ [:load])
-              |> Kernel.||(%{})
-              |> Map.get(join_path, %{})
-              |> Map.get(:data, [])
 
             destination_attribute_on_join_resource_type =
               Ash.Resource.Info.attribute(
@@ -1258,39 +1637,22 @@ defmodule Ash.Actions.Load do
     end
   end
 
-  defp true_load_query(relationship, query, data, path, request_path) do
+  defp true_load_query(relationship, query, data, source_data_path, join_relationship_path) do
-    {source_attribute, path} =
+    {source_attribute, source_data_path} =
-      if relationship.type == :many_to_many do
+      if relationship.type == :many_to_many && !lateral_join?(query, relationship, :unknown) do
-        join_relationship = join_relationship(relationship)
+        {relationship.destination_attribute_on_join_resource, join_relationship_path ++ [:data]}
-
-        {relationship.destination_attribute_on_join_resource,
-         join_relationship_path(path, join_relationship) |> Enum.map(& &1.name)}
       else
-        {relationship.source_attribute, path |> Enum.reverse() |> Enum.map(& &1.name)}
+        {relationship.source_attribute, source_data_path}
       end
 
-    source_data =
+    case get_in(data, source_data_path) do
-      case path do
-        [] ->
-          get_in(data, request_path ++ [:data, :results])
-
-        path ->
-          data
-          |> get_in(request_path)
-          |> Kernel.||(%{})
-          |> Map.get(:load, %{})
-          |> Map.get(path, %{})
-          |> Map.get(:data, %{})
-      end
-
-    case source_data do
       %{data: empty} when empty in [[], nil] ->
         :nothing
 
       empty when empty in [[], nil] ->
         :nothing
 
-      _ ->
+      source_data ->
         get_query(query, relationship, source_data, source_attribute)
     end
   end
@@ -1323,7 +1685,8 @@ defmodule Ash.Actions.Load do
           end
 
         ids =
-          Enum.flat_map(related_data, fn
+          related_data
+          |> Enum.flat_map(fn
             {_, data} when is_list(data) ->
               Enum.map(data, &Map.get(&1, source_attribute))
 

lib/ash/engine/engine.ex

@@ -367,8 +367,21 @@ defmodule Ash.Engine do
     end
   end
 
-  defp depends_on_summary(request, state) do
+  defp depends_on_summary(%{path: request_path} = request, state) do
-    dependencies = state.dependencies[request.path] || []
+    dependencies =
+      state.dependencies_waiting_on_request
+      |> Kernel.||([])
+      |> Enum.filter(fn
+        {^request_path, _} ->
+          true
+
+        _ ->
+          false
+      end)
+      |> Enum.map(fn {_, dep} ->
+        {:lists.droplast(dep), List.last(dep)}
+      end)
+      |> Enum.concat(state.dependencies[request_path] || [])
 
     if Enum.empty?(dependencies) do
       " state: #{request.state}"
@@ -378,7 +391,13 @@ defmodule Ash.Engine do
   end
 
   defp name_of({path, dep}, state) do
-    "#{inspect(Enum.find(state.requests, &(&1.path == path)).name)}.#{dep}"
+    case Enum.find(state.requests, &(&1.path == path)) do
+      nil ->
+        "unknown dependency: #{inspect(path, structs: false)} -> #{inspect(dep)}"
+
+      request ->
+        "#{request.name} -> #{inspect(dep)}"
+    end
   end
 
   defp run_iteration(

lib/ash/engine/request.ex

@@ -365,13 +365,13 @@ defmodule Ash.Engine.Request do
     end
   end
 
-  def summarize(%{name: name, action: %{name: action}, resource: resource})
+  def summarize(%{path: path, name: name, action: %{name: action}, resource: resource})
       when not is_nil(resource) do
-    "#{name}: #{inspect(resource)}.#{action}"
+    "#{inspect(path, structs: false)}\n#{name}: #{inspect(resource)}.#{action}"
   end
 
-  def summarize(%{name: name}) do
+  def summarize(%{path: path, name: name}) do
-    name
+    "#{inspect(path, structs: false)} - #{name}"
   end
 
   def sort_and_clean_notifications(notifications) do

lib/ash/error/exception.ex

@@ -16,7 +16,7 @@ defmodule Ash.Error.Exception do
                        error_context: [],
                        vars: [],
                        path: [],
-                       stacktrace: [],
+                       stacktrace: nil,
                        class: unquote(opts)[:class]
                      ]
 

lib/ash/filter/runtime.ex

@@ -44,10 +44,14 @@ defmodule Ash.Filter.Runtime do
 
     refs_to_load =
       refs_to_load
-      |> Enum.map(fn
+      |> Enum.reject(fn
-        %{attribute: %Ash.Resource.Calculation{load: nil} = calc} ->
+        %{attribute: %Ash.Resource.Calculation{}} ->
-          {calc.name, calc}
+          true
 
+        _ ->
+          false
+      end)
+      |> Enum.map(fn
         %{attribute: %{name: name}} ->
           name
       end)
@@ -537,10 +541,9 @@ defmodule Ash.Filter.Runtime do
            attribute: %Ash.Query.Calculation{
              module: module,
              opts: opts,
-             context: context,
+             context: context
-             name: name
            }
-         } = ref,
+         },
          record,
          parent,
          resource
@@ -573,13 +576,20 @@ defmodule Ash.Filter.Runtime do
         |> resolve_expr(record, parent, resource)
       end
     else
-      # This is problematic with variadic loads
+      # We need to rewrite this
-      resolve_ref(
+      # As it stands now, it will evaluate the calculation
-        %{ref | attribute: %Ash.Resource.Attribute{name: name}},
+      # once per expanded result. I'm not sure what that will
-        record,
+      # look like though.
-        parent,
+      case module.calculate([record], opts, context) do
-        resource
+        [result] ->
-      )
+          {:ok, result}
+
+        {:ok, [result]} ->
+          {:ok, result}
+
+        _ ->
+          {:ok, nil}
+      end
     end
   end
 

lib/ash/query/calculation.ex

@@ -63,6 +63,8 @@ defmodule Ash.Query.Calculation do
           :initial_limit,
           :initial_offset,
           :context,
+          :tenant,
+          :tracer,
           :ash
         ])
 

lib/ash/query/query.ex

@@ -863,7 +863,6 @@ defmodule Ash.Query do
   # Loading relationships with a query
   Ash.Query.load(query, [comments: [author: author_query]])
   ```
-
   """
   @spec load(
           t() | Ash.Resource.t(),
@@ -905,29 +904,8 @@ defmodule Ash.Query do
                      resource_calculation.filterable?,
                      resource_calculation.load
                    ) do
-              fields_to_select =
+              calculation =
-                resource_calculation.select
+                select_and_load_calc(resource_calculation, %{calculation | load: field}, query)
-                |> Kernel.||([])
-                |> Enum.concat(module.select(query, opts, calculation.context) || [])
-                |> Enum.uniq()
-                |> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
-
-              loads =
-                module.load(
-                  query,
-                  opts,
-                  Map.put(calculation.context, :context, query.context)
-                )
-                |> Ash.Actions.Helpers.validate_calculation_load!(module)
-                |> Enum.concat(List.wrap(resource_calculation.load))
-                |> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
-
-              calculation = %{
-                calculation
-                | load: field,
-                  select: fields_to_select,
-                  required_loads: loads
-              }
 
               Map.update!(query, :calculations, &Map.put(&1, field, calculation))
             end
@@ -941,34 +919,132 @@ defmodule Ash.Query do
     end)
   end
 
-  defp do_load(query, field) do
+  @doc false
-    cond do
+  def select_and_load_calc(resource_calculation, calculation, query) do
-      match?(%Ash.Query.Calculation{}, field) ->
+    module = calculation.module
-        calculation = field
+    opts = calculation.opts
+
+    resource_calculation_select =
+      if resource_calculation do
+        List.wrap(resource_calculation.select)
+      else
+        []
+      end
+
+    resource_calculation_load =
+      if resource_calculation do
+        List.wrap(resource_calculation.load)
+      else
+        []
+      end
 
     fields_to_select =
-          calculation.module.select(query, calculation.opts, calculation.context)
+      resource_calculation_select
       |> Kernel.||([])
+      |> Enum.concat(module.select(query, opts, calculation.context) || [])
       |> Enum.uniq()
       |> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
 
     loads =
-          calculation.module.load(
+      module.load(
         query,
-            calculation.opts,
+        opts,
         Map.put(calculation.context, :context, query.context)
       )
-          |> Ash.Actions.Helpers.validate_calculation_load!(calculation.module)
+      |> Ash.Actions.Helpers.validate_calculation_load!(module)
-          |> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
+      |> Enum.concat(resource_calculation_load)
+      |> reify_calculations(query)
 
-        calculation = %{
+    %{calculation | select: fields_to_select, required_loads: loads}
-          calculation
+  end
-          | load: field,
-            required_loads: loads,
-            select: fields_to_select
-        }
 
-        Map.update!(query, :calculations, &Map.put(&1, calculation.name, calculation))
+  @doc false
+  def reify_calculations(loads, query) do
+    loads
+    |> List.wrap()
+    |> Enum.map(fn
+      {load, args} ->
+        if resource_calculation = Ash.Resource.Info.calculation(query.resource, load) do
+          {module, opts} = resource_calculation.calculation
+
+          with {:ok, args} <- validate_calculation_arguments(resource_calculation, args),
+               {:ok, calculation} <-
+                 Calculation.new(
+                   resource_calculation.name,
+                   module,
+                   opts,
+                   {resource_calculation.type, resource_calculation.constraints},
+                   Map.put(args, :context, query.context),
+                   resource_calculation.filterable?,
+                   resource_calculation.load
+                 ) do
+            select_and_load_calc(resource_calculation, %{calculation | load: load}, query)
+          else
+            _ ->
+              {load, args}
+          end
+        else
+          if relationship = Ash.Resource.Info.relationship(query.resource, load) do
+            related_query = new(relationship.destination)
+            {load, reify_calculations(args, related_query)}
+          else
+            {load, args}
+          end
+        end
+
+      load ->
+        if resource_calculation = Ash.Resource.Info.calculation(query.resource, load) do
+          case resource_calc_to_calc(query, load, resource_calculation) do
+            {:error, _} ->
+              load
+
+            {:ok, calc} ->
+              calc
+          end
+        else
+          if relationship = Ash.Resource.Info.relationship(query.resource, load) do
+            {load, relationship.destination |> new() |> set_tenant(query.tenant)}
+          else
+            load
+          end
+        end
+    end)
+    |> case do
+      [%Ash.Query{} = query] -> query
+      other -> other
+    end
+  end
+
+  @doc false
+  def resource_calc_to_calc(query, name, resource_calculation) do
+    with %{calculation: {module, opts}} <- resource_calculation,
+         {:ok, args} <- validate_calculation_arguments(resource_calculation, %{}),
+         {:ok, calculation} <-
+           Calculation.new(
+             resource_calculation.name,
+             module,
+             opts,
+             {resource_calculation.type, resource_calculation.constraints},
+             Map.put(args, :context, query.context),
+             resource_calculation.filterable?,
+             resource_calculation.load
+           ) do
+      {:ok, select_and_load_calc(resource_calculation, %{calculation | load: name}, query)}
+    end
+  end
+
+  defp do_load(query, field) do
+    cond do
+      match?(%Ash.Query.Calculation{}, field) ->
+        Map.update!(
+          query,
+          :calculations,
+          &Map.put(
+            &1,
+            field.name,
+            select_and_load_calc(nil, field, query)
+          )
+        )
 
       Ash.Resource.Info.attribute(query.resource, field) ->
         Ash.Query.ensure_selected(query, field)
@@ -1038,47 +1114,12 @@ defmodule Ash.Query do
         end
 
       resource_calculation = Ash.Resource.Info.calculation(query.resource, field) ->
-        {module, opts} = resource_calculation.calculation
+        case resource_calc_to_calc(query, resource_calculation.name, resource_calculation) do
-
-        with {:ok, args} <- validate_calculation_arguments(resource_calculation, %{}),
-             {:ok, calculation} <-
-               Calculation.new(
-                 resource_calculation.name,
-                 module,
-                 opts,
-                 {resource_calculation.type, resource_calculation.constraints},
-                 Map.put(args, :context, query.context),
-                 resource_calculation.filterable?,
-                 resource_calculation.load
-               ) do
-          fields_to_select =
-            resource_calculation.select
-            |> Kernel.||([])
-            |> Enum.concat(module.select(query, opts, calculation.context) || [])
-            |> Enum.uniq()
-            |> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
-
-          loads =
-            module.load(
-              query,
-              opts,
-              Map.put(calculation.context, :context, query.context)
-            )
-            |> Ash.Actions.Helpers.validate_calculation_load!(module)
-            |> Enum.concat(List.wrap(resource_calculation.load))
-            |> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
-
-          calculation = %{
-            calculation
-            | load: field,
-              required_loads: loads,
-              select: fields_to_select
-          }
-
-          Map.update!(query, :calculations, &Map.put(&1, field, calculation))
-        else
           {:error, error} ->
             add_error(query, :load, error)
+
+          {:ok, calc} ->
+            Map.update!(query, :calculations, &Map.put(&1, field, calc))
         end
 
       true ->
@@ -1705,7 +1746,7 @@ defmodule Ash.Query do
           )
           |> Ash.Actions.Helpers.validate_calculation_load!(module)
           |> Enum.concat(List.wrap(calculation.required_loads))
-          |> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
+          |> reify_calculations(query)
 
         calculation = %{calculation | select: fields_to_select, required_loads: loads}
         %{query | calculations: Map.put(query.calculations, name, calculation)}
@@ -1735,30 +1776,12 @@ defmodule Ash.Query do
                resource_calculation.filterable?,
                resource_calculation.load
              ) do
-        fields_to_select =
+        calculation =
-          resource_calculation.select
+          select_and_load_calc(
-          |> Kernel.||([])
+            resource_calculation,
-          |> Enum.concat(module.select(query, opts, calculation.context) || [])
+            %{calculation | load: nil, name: as_name},
-          |> Enum.uniq()
+            query
-          |> Enum.filter(&Ash.Resource.Info.attribute(query.resource, &1))
-
-        loads =
-          module.load(
-            query,
-            opts,
-            Map.put(calculation.context, :context, query.context)
           )
-          |> Ash.Actions.Helpers.validate_calculation_load!(module)
-          |> Enum.concat(List.wrap(resource_calculation.load))
-          |> Enum.reject(&Ash.Resource.Info.attribute(query.resource, &1))
-
-        calculation = %{
-          calculation
-          | name: as_name,
-            load: nil,
-            select: fields_to_select,
-            required_loads: loads
-        }
 
         Map.update!(query, :calculations, &Map.put(&1, as_name, calculation))
       else
@@ -2103,24 +2126,31 @@ defmodule Ash.Query do
 
     keys
     |> Enum.reduce(query, fn key, query ->
-      if key in [:api, :resource] do
+      do_unset(query, key, new)
-        query
-      else
-        struct(query, [{key, Map.get(new, key)}])
-      end
     end)
   end
 
   def unset(query, key) do
-    if key in [:api, :resource] do
-      to_query(query)
-    else
     new = new(query.resource)
+    do_unset(query, key, new)
 
     query
     |> to_query()
     |> struct([{key, Map.get(new, key)}])
   end
+
+  defp do_unset(query, key, _new) when key in [:api, :resource] do
+    query
+  end
+
+  defp do_unset(query, :load, new) do
+    query = unset(query, [:calculations, :aggregates])
+
+    struct(query, [{:load, Map.get(new, :load)}])
+  end
+
+  defp do_unset(query, key, new) do
+    struct(query, [{key, Map.get(new, key)}])
   end
 
   @doc "Return the underlying data layer query for an ash query"

lib/ash/resource/manual_relationship/manual_relationship.ex

@@ -6,7 +6,6 @@ defmodule Ash.Resource.ManualRelationship do
   @type context :: %{
           relationship: Ash.Resource.Relationships.relationship(),
           query: Ash.Query.t(),
-          root_query: Ash.Query.t(),
           actor: term,
           tenant: term,
           authorize?: term,

lib/ash/type/function.ex

@@ -4,7 +4,7 @@ defmodule Ash.Type.Function do
 
   If the type would be dumped to a native format, `:erlang.term_to_binary(term, [:safe])` is used.
 
-  Please keep in mind, this is not safe to use with external input. This could easily cause you t
+  Please keep in mind, this is *NOT SAFE* to use with external input.
 
   More information available here: https://erlang.org/doc/man/erlang.html#binary_to_term-2
   """

read-action-refactor.md

@@ -0,0 +1,129 @@
+# Read Actions & Data Loading Refactor
+
+Problems:
+
+1. Loading data - lots of expensive comparison operations being performed to check what's been loaded, etc.
+  - calculation metadata is recursively populated
+  - no simple format for declaring the data that needs to be loaded
+2. Read action flow
+
+
+Idea: use Ash.Flow to implement read (and in future all) actions.
+Needs:
+  1. First class ability of steps to generate new steps.
+  2. (Maybe) ergonomics improvements to some conditional steps, eg: `branch` rather than `if/else`.
+  3. Maybe conditional dependencies?
+
+# Actions:
+1. Feature flag for switching between `read.ex` and `read_flow.ex`.
+2. Iterate until all the tests pass.
+
+# Suspicions:
+1. Ash.Flow step generator.
+2. Ash.Flow if/else steps.
+3. Conditional dependencies?
+
+
+
+This is too big to fail, so: let's just do calculations.
+
+New structure for calculations:
+
+Benefits:
+
+simpler to manipulate and check what calculations have been loaded, and comparing what calculations have been loaded no longer *also must* compare their definitions.
+
+Note:
+we end up removing `name` and `load` from `Ash.Query.Calculation` (maybe eventually? In Ash 3.0? If we can achieve backwards compatibility then we'd leave them. Maybe won't matter, needs to be thought about)
+
+Filters will be fine without the keys above because they don't need to call them anything to put them into an expression
+
+
+```elixir
+  @type calculation_definition :: %{name: atom(), target: target()}
+  @type target :: {:calculations, atom()} | {:top_level, atom()}
+  # better names for this key to be workshopped
+  calculations_to_load: %{
+    score1: %{args: %{}, load_as: :}
+    full_name: %{arg1: 10},
+    full_name: %{arg1: 11},
+    full_name: %{arg1: 12},
+    some_random_shit: %{arg1: 10}
+  },
+  calculation_definitions: %{
+    # Supports anonymous calculations & resource calculations
+    %{name: :full_name, target: {:top_level, :full_name}} => %Ash.Query.Calculation{},
+    %{name: :full_name, target: {:calculations, :full_name_2}} => %Ash.Query.Calculation{}
+  }
+```
+
+Scratch
+
+```elixir
+  %{
+    full_name: %Calculation{},
+    some_random_shit: %Calculation{name: :some_random_shit}
+  }
+
+  %Calculation{name: :full_name, load: , context: %{arg1: 10, actor: ..., ...., ...}}
+  
+
+  calculations do
+    calculate :full_name, :string, 
+      expr(first_name <> ^arg(:separator) <> last_name) do
+     argument :separator, :string do
+          allow_nil? false
+          default " "
+        end
+    end
+  end
+
+  User
+  |> Ash.Query.load(full_name: %{separator: "~"})
+  |> Ash.Query.calculate(:thing, Ash.Query.Calculation.new(...))
+
+  %Resource{
+    thing: thing,
+    calculations: %{
+      thing2: thing
+    }
+  }
+  %{name: :thing, target: :thing}
+  %{name: :thing, target: {:calculations, :thing2}}
+
+  calculations: %{
+    full_name: %{args: %{separator: "~"}}
+  },
+  calculation_definitions: %{
+    %{name: :full_name, load: :full_name} => %Ash.Query.Calculation{}
+  }
+
+  %Resource{
+    name: :value,
+    calculations: %{}
+  }
+  
+  Ash.Resource.Info.calculation(query.resource, calc_name,)
+  
+  some_read_action
+  |> read_them()
+  |> Enum.map(...)
+  |> calculation(....)
+
+  
+  query
+  |> Ash.Query.load_calculation_as(:score, :score_1, %{arg: 1})
+  |> Ash.Query.load_calculation_as(:score, :score_1, %{arg: 1})
+```
+
+```elixir
+# What calculate looks like now, context jumbled in with context
+def calculate(records, opts, %{arg1: arg1, actor: actor}) do
+
+end
+
+# What it should look like < 3.0 >
+def calculate(records, opts, args, %CalculationContext{actor: actor, tenant: tenant, authorize?: authorize?} = context) do
+
+end
+```

test/actions/async_load_test.exs

@@ -219,8 +219,6 @@ defmodule Ash.Test.Actions.AsyncLoadTest do
       |> manage_relationship(:author, author, type: :append_and_remove)
       |> Api.create!()
 
-      Application.put_env(:foo, :bar, true)
-
       authorized_posts =
         author
         |> Api.load!(:authorized_actor_posts)

test/calculation_test.exs

@@ -4,6 +4,31 @@ defmodule Ash.Test.CalculationTest do
 
   require Ash.Query
 
+  defmodule FriendLink do
+    use Ash.Resource,
+      data_layer: Ash.DataLayer.Ets
+
+    ets do
+      private? true
+    end
+
+    actions do
+      defaults [:create, :read, :update, :destroy]
+    end
+
+    relationships do
+      belongs_to :source, Ash.Test.CalculationTest.Friend do
+        allow_nil? false
+        primary_key? true
+      end
+
+      belongs_to :target, Ash.Test.CalculationTest.Friend do
+        allow_nil? false
+        primary_key? true
+      end
+    end
+  end
+
   defmodule Concat do
     # An example concatenation calculation, that accepts the delimiter as an argument
     use Ash.Calculation
@@ -89,9 +114,9 @@ defmodule Ash.Test.CalculationTest do
     def load(_query, _opts, args) do
       if args[:only_special] do
         query =
-          __MODULE__.User
+          Ash.Test.CalculationTest.User
           |> Ash.Query.filter(special == true)
-          |> Ash.Query.ensure_selected(:full_name)
+          |> Ash.Query.load(:full_name)
 
         [best_friends_of_me: query]
       else
@@ -123,6 +148,32 @@ defmodule Ash.Test.CalculationTest do
     end
   end
 
+  defmodule FriendsNames do
+    use Ash.Calculation
+
+    def load(_query, _opts, _) do
+      [
+        friends:
+          Ash.Test.CalculationTest.User
+          |> Ash.Query.load([:full_name, :prefix])
+          |> Ash.Query.limit(5)
+      ]
+    end
+
+    def calculate(records, _opts, _) do
+      Enum.map(records, fn record ->
+        record.friends
+        |> Enum.map_join(" | ", fn friend ->
+          if friend.prefix do
+            friend.prefix <> " " <> friend.full_name
+          else
+            friend.full_name
+          end
+        end)
+      end)
+    end
+  end
+
   defmodule User do
     @moduledoc false
     use Ash.Resource, data_layer: Ash.DataLayer.Ets
@@ -146,6 +197,7 @@ defmodule Ash.Test.CalculationTest do
       uuid_primary_key :id
       attribute :first_name, :string
       attribute :last_name, :string
+      attribute :prefix, :string
       attribute :special, :boolean
     end
 
@@ -171,7 +223,8 @@ defmodule Ash.Test.CalculationTest do
                 :string,
                 {ConcatWithLoad, keys: [:full_name, :full_name_plus_full_name]}
 
-      calculate :slug, :string, expr(full_name <> "123"), load: [:full_name]
+      calculate :slug, :string, expr(full_name <> "123")
+      calculate :friends_names, :string, FriendsNames
 
       calculate :expr_full_name, :string, expr(first_name <> " " <> last_name)
 
@@ -181,7 +234,10 @@ defmodule Ash.Test.CalculationTest do
 
       calculate :best_friends_name, :string, BestFriendsName
 
-      calculate :names_of_best_friends_of_me, :string, NamesOfBestFriendsOfMe
+      calculate :names_of_best_friends_of_me, :string, NamesOfBestFriendsOfMe do
+        argument :only_special, :boolean, default: false
+      end
+
       calculate :name_with_users_name, :string, NameWithUsersName
 
       calculate :full_name_with_salutation,
@@ -237,6 +293,12 @@ defmodule Ash.Test.CalculationTest do
       has_many :best_friends_of_me, __MODULE__ do
         destination_attribute :best_friend_id
       end
+
+      many_to_many :friends, __MODULE__ do
+        through FriendLink
+        destination_attribute_on_join_resource :target_id
+        source_attribute_on_join_resource :source_id
+      end
     end
   end
 
@@ -245,7 +307,8 @@ defmodule Ash.Test.CalculationTest do
     use Ash.Registry
 
     entries do
-      entry(User)
+      entry User
+      entry FriendLink
     end
   end
 
@@ -268,6 +331,7 @@ defmodule Ash.Test.CalculationTest do
       User
       |> Ash.Changeset.new(%{first_name: "brian", last_name: "cranston"})
       |> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
+      |> Ash.Changeset.manage_relationship(:friends, user1, type: :append_and_remove)
       |> Api.create!()
 
     %{user1: user1, user2: user2}
@@ -321,26 +385,10 @@ defmodule Ash.Test.CalculationTest do
            ]
   end
 
-  test "it doesn't reload anything specified by the load callback if its already been loaded when using `lazy?: true`" do
-    full_names =
-      User
-      |> Ash.Query.load(:full_name_plus_full_name)
-      |> Api.read!()
-      |> Enum.map(&%{&1 | full_name: &1.full_name <> " more"})
-      |> Api.load!(:full_name_plus_full_name, lazy?: true)
-      |> Enum.map(& &1.full_name_plus_full_name)
-      |> Enum.sort()
-
-    assert full_names == [
-             "brian cranston more brian cranston more",
-             "zach daniel more zach daniel more"
-           ]
-  end
-
   test "it reloads anything specified by the load callback if its already been loaded when using `lazy?: false`" do
     full_names =
       User
-      |> Ash.Query.load(:full_name_plus_full_name)
+      |> Ash.Query.load([:full_name, :full_name_plus_full_name])
       |> Api.read!()
       |> Enum.map(&%{&1 | full_name: &1.full_name <> " more"})
       |> Api.load!(:full_name_plus_full_name)
@@ -515,34 +563,97 @@ defmodule Ash.Test.CalculationTest do
     assert full_names == ["bob", "brian cranston", "zach daniel"]
   end
 
-  # test "loading calculations with different relationship dependencies won't collide", %{
+  test "loading calculations with different relationship dependencies won't collide", %{
-  #   user1: %{id: user1_id} = user1
+    user1: %{id: user1_id} = user1
-  # } do
+  } do
-  #   user3 =
+    User
-  #     User
+    |> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
-  #     |> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
+    |> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
-  #     |> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
+    |> Api.create!()
-  #     |> Api.create!()
 
-  #   assert %{
+    assert %{
-  #            calculations: %{
+             calculations: %{
-  #              names_of_best_friends_of_me: "brian cranston - chidi anagonye",
+               names_of_best_friends_of_me: "brian cranston - chidi anagonye",
-  #              names_of_special_best_friends_of_me: "chidi anagonye"
+               names_of_special_best_friends_of_me: "chidi anagonye"
-  #            }
+             }
-  #          } =
+           } =
-  #            User
+             User
-  #            |> Ash.Query.filter(id == ^user1_id)
+             |> Ash.Query.filter(id == ^user1_id)
-  #            |> Ash.Query.load_calculation_as(
+             |> Ash.Query.load_calculation_as(
-  #              :names_of_best_friends_of_me,
+               :names_of_best_friends_of_me,
-  #              :names_of_special_best_friends_of_me,
+               :names_of_special_best_friends_of_me,
-  #              %{
+               %{
-  #                special: true
+                 only_special: true
-  #              }
+               }
-  #            )
+             )
-  #            |> Ash.Query.load_calculation_as(
+             |> Ash.Query.load_calculation_as(
-  #              :names_of_best_friends_of_me,
+               :names_of_best_friends_of_me,
-  #              :names_of_best_friends_of_me
+               :names_of_best_friends_of_me
-  #            )
+             )
-  #            |> Api.read_one!()
+             |> Api.read_one!()
-  # end
+  end
+
+  test "loading calculations with different relationship dependencies won't collide, when manually loading one of the deps",
+       %{
+         user1: %{id: user1_id} = user1
+       } do
+    User
+    |> Ash.Changeset.new(%{first_name: "chidi", last_name: "anagonye", special: true})
+    |> Ash.Changeset.manage_relationship(:best_friend, user1, type: :append_and_remove)
+    |> Api.create!()
+
+    assert %{
+             calculations: %{
+               names_of_best_friends_of_me: "brian cranston - chidi anagonye",
+               names_of_special_best_friends_of_me: "chidi anagonye"
+             }
+           } =
+             User
+             |> Ash.Query.filter(id == ^user1_id)
+             |> Ash.Query.load(:best_friends_of_me)
+             |> Ash.Query.load_calculation_as(
+               :names_of_best_friends_of_me,
+               :names_of_special_best_friends_of_me,
+               %{
+                 only_special: true
+               }
+             )
+             |> Ash.Query.load_calculation_as(
+               :names_of_best_friends_of_me,
+               :names_of_best_friends_of_me
+             )
+             |> Api.read_one!()
+  end
+
+  test "calculations that depend on many to many relationships will load", %{user2: user2} do
+    assert %{
+             friends_names: "zach daniel"
+           } =
+             User
+             |> Ash.Query.filter(id == ^user2.id)
+             |> Ash.Query.load(:friends_names)
+             |> Api.read_one!()
+  end
+
+  test "when already loading a calculation's dependency it is used" do
+    full_names =
+      User
+      |> Ash.Query.load([:full_name, :full_name_plus_full_name])
+      |> Api.read!()
+      |> Enum.map(& &1.full_name_plus_full_name)
+      |> Enum.sort()
+
+    assert full_names == ["brian cranston brian cranston", "zach daniel zach daniel"]
+  end
+
+  test "when already loading a nested calculation dependency, it is used" do
+    best_friends_names =
+      User
+      |> Ash.Query.load([:best_friends_name, best_friend: [:full_name]])
+      |> Api.read!()
+      |> Enum.map(& &1.best_friends_name)
+      |> Enum.sort()
+
+    assert best_friends_names == [nil, "zach daniel"]
+  end
 end