improvement: add optimized path for casting embeds when they are simple

improvement: add `include_embedded_source_by_default?` config to optimize embeds
improvement: support `:fields` constraint on `:struct` type, enabling persistence
This commit is contained in:
Zach Daniel 2024-08-07 16:45:46 -04:00
parent 1711ecf574
commit a719c791ba
9 changed files with 686 additions and 131 deletions

View file

@ -1,9 +1,17 @@
defmodule Domain do
use Ash.Domain, validate_config_inclusion?: false
resources do
allow_unregistered? true
end
end
defmodule Embed do
use Ash.Resource, data_layer: :embedded
attributes do
# uuid_primary_key :id
attribute :name, :string
attribute :name, :string, public?: true
end
actions do
@ -17,8 +25,21 @@ defmodule Resource do
attributes do
uuid_primary_key :id
attribute :embeds, {:array, Embed}
attribute :maps, {:array, :map}
attribute :embeds, {:array, Embed}, public?: true
attribute :structs, {:array, :struct} do
public? true
constraints [
items: [
instance_of: Embed,
fields: [
name: [
type: :string
]
]
]
]
end
attribute :maps, {:array, :map}, public?: true
end
actions do
@ -27,16 +48,20 @@ defmodule Resource do
end
end
defmodule Domain do
use Ash.Domain
resources do
resource Resource
end
end
embeds_input = 1..100 |> Enum.map(&%{name: "Embed #{&1}"})
resource_embeds_input =
1..100
|> Enum.map(fn _ -> %{embeds: embeds_input} end)
resource_maps_input =
1..100
|> Enum.map(fn _ -> %{maps: embeds_input} end)
resource_structs_input =
1..100
|> Enum.map(fn _ -> %{structs: embeds_input} end)
Resource
|> Ash.Changeset.for_create(:create, %{embeds: embeds_input, maps: embeds_input})
|> Ash.create!()
@ -44,14 +69,14 @@ Resource
Benchee.run(
%{
embeds: fn ->
Resource
|> Ash.Changeset.for_create(:create, %{embeds: embeds_input})
|> Ash.create!()
Ash.bulk_create!(resource_embeds_input, Resource, :create)
end,
maps: fn ->
Resource
|> Ash.Changeset.for_create(:create, %{maps: embeds_input})
|> Ash.create!()
Ash.bulk_create!(resource_maps_input, Resource, :create)
end,
structs: fn ->
Ash.bulk_create!(resource_structs_input, Resource, :create)
end
}
},
memory_time: 2
)

View file

@ -236,7 +236,7 @@ defmodule Helpdesk.Support.Ticket do
end
```
Next, add your domain to your `config.exs`
Next, add your domain to your `config.exs`, and configure some backwards compatibility configuration.
Run the following to create your `config.exs` if it doesn't already exist
@ -252,6 +252,8 @@ and add the following contents to it.
import Config
config :helpdesk, :ash_domains, [Helpdesk.Support]
config :include_embedded_source_by_default?, true
```
### Try our first resource out

View file

@ -1,6 +1,12 @@
defmodule Ash.EmbeddableType do
@moduledoc false
@include_source_by_default Application.compile_env(
:ash,
:include_embedded_source_by_default?,
true
)
@embedded_resource_array_constraints [
sort: [
type: :any,
@ -56,6 +62,12 @@ defmodule Ash.EmbeddableType do
type: :atom,
doc: "The read action to use when reading the embed. The primary is used by default."
],
include_source?: [
type: :boolean,
default: @include_source_by_default,
doc:
"Whether to include the source changeset in the context. Defaults to the value of `config :ash, :include_embedded_source_by_default`, or `true`. In 4.x, the default will be `false`."
],
__source__: [
type: :any,
hide: true,
@ -144,6 +156,11 @@ defmodule Ash.EmbeddableType do
defmacro single_embed_implementation(opts) do
# credo:disable-for-next-line Credo.Check.Refactor.LongQuoteBlocks
quote generated: true, location: :keep, bind_quoted: [opts: opts] do
@include_source_by_default Application.compile_env(
:ash,
:include_embedded_source_by_default?,
true
)
alias Ash.EmbeddableType.ShadowDomain
def storage_type(_), do: :map
@ -199,56 +216,156 @@ defmodule Ash.EmbeddableType do
def cast_input_array(value, constraints) when is_list(value) do
action =
constraints[:create_action] ||
Ash.Resource.Info.primary_action!(__MODULE__, :create).name
case constraints[:create_action] do
nil ->
Ash.Resource.Info.primary_action!(__MODULE__, :create)
action ->
Ash.Resource.Info.action(__MODULE__, action)
end
{structs, values} =
value
|> Stream.with_index()
|> Enum.split_with(&is_struct(elem(&1, 0), __MODULE__))
{context, opts} =
case constraints[:__source__] do
%Ash.Changeset{context: context} = source ->
{Map.put(context, :__source__, source),
Ash.Context.to_opts(context[:private] || %{})}
if Enum.empty?(values) do
{:ok, Enum.map(structs, &elem(&1, 0))}
else
skip_unknown_inputs = List.wrap(constraints[:skip_unknown_inputs])
_ ->
{%{}, []}
end
# This is a simplified, tight-loop version of resource creation
if !constraints[:include_source?] &&
Enum.empty?(action.changes) &&
Enum.empty?(Ash.Resource.Info.changes(__MODULE__, action.type)) &&
Enum.empty?(Ash.Resource.Info.validations(__MODULE__, action.type)) &&
Enum.empty?(Ash.Resource.Info.notifiers(__MODULE__)) &&
Enum.empty?(Ash.Resource.Info.relationships(__MODULE__)) do
Enum.reduce_while(values, {:ok, []}, fn {value, index}, {:ok, results} ->
Enum.reduce_while(value, {:ok, index, %{__struct__: __MODULE__}}, fn {key, value},
{:ok, index,
acc} ->
case Ash.Resource.Info.attribute(__MODULE__, key) do
nil ->
{:cont, {:ok, acc}}
values
|> Stream.map(&elem(&1, 0))
|> Ash.bulk_create(
__MODULE__,
action,
Keyword.merge(opts,
domain: ShadowDomain,
context: context,
sorted?: true,
skip_unknown_inputs: skip_unknown_inputs(constraints),
return_records?: true,
return_errors?: true,
batch_size: 1_000_000_000
)
)
|> case do
%{status: :success, records: records} ->
Enum.reduce(structs, {:ok, records}, fn {struct, index}, {:ok, records} ->
{:ok, List.insert_at(records, index, struct)}
end)
%{errors: errors} ->
errors =
Enum.map(errors, fn
%Ash.Changeset{context: %{bulk_create: %{index: index}}, errors: errors} ->
Ash.Error.set_path(Ash.Error.to_ash_error(errors), index)
other ->
Ash.Error.to_ash_error(other)
attribute ->
if attribute.name in action.accept do
with value <- Ash.Type.Helpers.handle_indexed_maps(attribute.type, value),
{:ok, casted} <-
Ash.Type.cast_input(attribute.type, value, attribute.constraints),
{:ok, casted} <-
Ash.Type.apply_constraints(
attribute.type,
casted,
attribute.constraints
) do
{:cont, {:ok, index, Map.put(acc, key, casted)}}
else
{:error, error} ->
{:halt, {:error, index, error}}
end
else
if Enum.any?(skip_unknown_inputs, &(&1 == :* || &1 == key)) do
{:cont, {:ok, index, acc}}
else
{:halt,
{:error, index,
Ash.Error.Invalid.NoSuchInput.exception(
resource: __MODULE__,
action: action.name,
input: key,
inputs: Ash.Resource.Info.action_inputs(__MODULE__, action.name)
)}}
end
end
end
end)
|> case do
{:ok, index, result} ->
__MODULE__
|> Ash.Resource.Info.attributes()
|> Stream.filter(
&(&1.name not in action.allow_nil_input &&
(&1.allow_nil? == false || &1.name in action.require_attributes))
)
|> Enum.reduce_while({:ok, result}, fn attr, {:ok, result} ->
if is_nil(Map.get(result, attr.name)) do
{:halt,
{:error,
Ash.Error.Changes.Required.exception(
resource: __MODULE__,
field: attr.name,
type: :attribute
)}}
else
{:cont, {:ok, result}}
end
end)
|> case do
{:ok, result} ->
{:cont, {:ok, [result | results]}}
{:error, Ash.Error.to_ash_error(errors)}
{:error, error} ->
{:halt, {:error, index, error}}
end
{:error, index, error} ->
{:halt, {:error, index, error}}
end
end)
|> case do
{:ok, values} ->
{:ok, Enum.reverse(values)}
{:error, index, error} ->
{:error, Ash.Error.set_path(Ash.Error.to_ash_error(error), index)}
end
else
{context, opts} =
case constraints[:__source__] do
%Ash.Changeset{context: context} = source ->
{Map.put(context, :__source__, source),
Ash.Context.to_opts(context[:private] || %{})}
_ ->
{%{}, []}
end
values
|> Stream.map(&elem(&1, 0))
|> Ash.bulk_create(
__MODULE__,
action.name,
Keyword.merge(opts,
domain: ShadowDomain,
context: context,
sorted?: true,
skip_unknown_inputs: skip_unknown_inputs(constraints),
return_records?: true,
return_errors?: true,
batch_size: 1_000_000_000
)
)
|> case do
%{status: :success, records: records} ->
Enum.reduce(structs, {:ok, records}, fn {struct, index}, {:ok, records} ->
{:ok, List.insert_at(records, index, struct)}
end)
%{errors: errors} ->
errors =
Enum.map(errors, fn
%Ash.Changeset{context: %{bulk_create: %{index: index}}, errors: errors} ->
Ash.Error.set_path(Ash.Error.to_ash_error(errors), index)
other ->
Ash.Error.to_ash_error(other)
end)
{:error, Ash.Error.to_ash_error(errors)}
end
end
end
end
@ -806,7 +923,11 @@ defmodule Ash.EmbeddableType do
end
def include_source(constraints, changeset) do
Keyword.put(constraints, :__source__, changeset)
if Keyword.get(constraints, :include_source?, @include_source_by_default) do
Keyword.put(constraints, :__source__, changeset)
else
constraints
end
end
def prepare_change_array(_old_values, nil, _constraints) do

View file

@ -1 +0,0 @@

View file

@ -107,8 +107,8 @@ defmodule Ash.Type.Map do
@impl true
def cast_atomic(new_value, constraints) do
if constraints[:keys] do
{:not_atomic, "Keywords do not support atomic updates when using the `keys` constraint"}
if constraints[:fields] do
{:not_atomic, "Maps do not support atomic updates when using the `fields` constraint"}
else
{:atomic, new_value}
end

View file

@ -1,19 +1,79 @@
defmodule Ash.Type.Struct do
@moduledoc """
Represents a struct.
This cannot be loaded from a database, it can only be used to cast input.
Use the `instance_of` constraint to specify that it must be an instance of a specific struct.
"""
use Ash.Type
@constraints [
instance_of: [
type: :atom,
doc: "The module the struct should be an instance of"
],
preserve_nil_values?: [
type: :boolean,
default: false,
doc:
"If set to true, when storing, nil values will be kept. Otherwise, nil values will be omitted."
],
fields: [
type: :keyword_list,
keys: [
*: [
type: :keyword_list,
keys: [
type: [
type: Ash.OptionsHelpers.ash_type(),
required: true
],
allow_nil?: [
type: :boolean,
default: true
],
constraints: [
type: :keyword_list,
default: []
]
]
]
],
doc: """
The types of the fields in the struct, and their constraints.
For example:
fields: [
amount: [
type: :integer,
constraints: [
max: 10
]
],
currency: [
type: :string,
allow_nil?: false,
constraints: [
max_length: 3
]
]
]
allow_nil? is true by default
"""
]
]
@moduledoc """
Represents a struct.
Use the `instance_of` constraint to specify that it must be an instance of a specific struct.
This cannot be loaded from a database unless the `instance_of` constraint is provided.
If not, it can only be used to cast input, i.e for arguments.
## Constraints
#{Spark.Options.docs(@constraints)}
"""
def field_types(value) do
{:ok, value}
end
use Ash.Type
@impl true
def constraints, do: @constraints
@ -22,77 +82,227 @@ defmodule Ash.Type.Struct do
def storage_type(_), do: :map
@impl true
def matches_type?(v, constraints) do
if instance_of = constraints[:instance_of] do
is_struct(v, instance_of)
else
is_struct(v)
end
end
@impl true
def cast_input("", _), do: {:ok, nil}
def cast_input(nil, _), do: {:ok, nil}
def cast_input(%struct{} = value, constraints) do
case constraints[:instance_of] do
nil ->
{:ok, value}
^struct ->
{:ok, value}
def cast_input(value, constraints) when is_binary(value) do
case Jason.decode(value) do
{:ok, value} ->
cast_input(value, constraints)
_ ->
:error
end
end
def cast_input(value, _) when is_map(value), do: {:ok, value}
def cast_input(_, _), do: :error
@impl true
def matches_type?(v, _) do
is_struct(v)
end
@impl Ash.Type
def load(record, load, _constraints, %{domain: domain} = context) do
opts = Ash.Context.to_opts(context, domain: domain)
Ash.load(record, load, opts)
end
@impl Ash.Type
def merge_load(left, right, constraints, context) do
instance_of = constraints[:instance_of]
left = Ash.Query.load(instance_of, left)
right = Ash.Query.load(instance_of, right)
if left.valid? do
{:ok, Ash.Query.merge_query_load(left, right, context)}
else
{:error, Ash.Error.to_ash_error(left.errors)}
end
end
@impl Ash.Type
def get_rewrites(merged_load, calculation, path, constraints) do
instance_of = constraints[:instance_of]
if instance_of && Ash.Resource.Info.resource?(instance_of) do
merged_load = Ash.Query.load(instance_of, merged_load)
Ash.Actions.Read.Calculations.get_all_rewrites(merged_load, calculation, path)
else
[]
end
end
@impl Ash.Type
def rewrite(value, rewrites, _constraints) do
Ash.Actions.Read.Calculations.rewrite(rewrites, value)
end
@impl Ash.Type
def can_load?(constraints) do
instance_of = constraints[:instance_of]
instance_of && Ash.Resource.Info.resource?(instance_of)
end
@impl true
def cast_stored(nil, _), do: {:ok, nil}
def cast_stored(value, constraints) when is_map(value) do
if fields = constraints[:fields] do
if constraints[:instance_of] do
nil_values = constraints[:store_nil_values?]
Enum.reduce_while(fields, {:ok, %{}}, fn {key, config}, {:ok, acc} ->
case Map.fetch(value, key) do
{:ok, value} ->
case Ash.Type.cast_stored(config[:type], value, config[:constraints] || []) do
{:ok, value} ->
if is_nil(value) && !nil_values do
{:cont, {:ok, acc}}
else
{:cont, {:ok, Map.put(acc, key, value)}}
end
other ->
{:halt, other}
end
:error ->
{:cont, {:ok, acc}}
end
end)
else
:error
end
else
:error
end
end
def cast_stored(_, _), do: :error
@impl true
def dump_to_native(nil, _), do: {:ok, nil}
def dump_to_native(value, constraints) when is_map(value) do
if fields = constraints[:fields] do
if constraints[:instance_of] do
Enum.reduce_while(fields, {:ok, %{}}, fn {key, config}, {:ok, acc} ->
case Map.fetch(value, key) do
{:ok, value} ->
case Ash.Type.dump_to_native(config[:type], value, config[:constraints] || []) do
{:ok, value} ->
{:cont, {:ok, Map.put(acc, key, value)}}
other ->
{:halt, other}
end
:error ->
{:cont, {:ok, acc}}
end
end)
else
:error
end
else
:error
end
end
def dump_to_native(_, _), do: :error
@impl true
def cast_atomic(new_value, constraints) do
if constraints[:fields] do
{:not_atomic, "Structs do not support atomic updates when using the `keys` constraint"}
else
{:atomic, new_value}
end
end
@impl true
def apply_constraints(value, constraints) do
with {:ok, value} <- handle_fields(value, constraints) do
handle_instance_of(value, constraints)
end
end
defp handle_fields(value, constraints) do
case Keyword.fetch(constraints, :fields) do
{:ok, fields} when is_list(fields) ->
check_fields(value, fields)
_ ->
{:ok, value}
end
end
defp handle_instance_of(value, constraints) do
case Keyword.fetch(constraints, :instance_of) do
{:ok, struct} ->
cond do
is_struct(value, struct) ->
{:ok, value}
is_struct(value) ->
:error
value ->
if constraints[:fields] do
{:ok, struct(struct, value)}
else
keys = Map.keys(value)
cond do
Enum.all?(keys, &is_atom/1) ->
{:ok, struct(struct, value)}
Enum.all?(keys, &is_binary/1) ->
{:ok, Map.delete(struct.__struct__, :__struct__)}
|> Enum.reduce({:ok, struct(struct)}, fn {key, _value}, {:ok, acc} ->
case Map.fetch(value, to_string(key)) do
{:ok, val} ->
{:ok, Map.put(acc, key, val)}
:error ->
{:ok, acc}
end
end)
end
end
end
:error ->
if is_struct(value) do
{:ok, value}
else
:error
end
end
end
defp check_fields(value, fields) do
Enum.reduce(fields, {:ok, %{}}, fn
{field, field_constraints}, {:ok, checked_value} ->
case fetch_field(value, field) do
{:ok, field_value} ->
check_field(checked_value, field, field_value, field_constraints)
:error ->
if field_constraints[:allow_nil?] == false do
{:error, [[message: "field must be present", field: field]]}
else
{:ok, checked_value}
end
end
{_, _}, {:error, errors} ->
{:error, errors}
end)
end
defp check_field(result, field, field_value, field_constraints) do
case Ash.Type.cast_input(
field_constraints[:type],
field_value,
field_constraints[:constraints] || []
) do
{:ok, field_value} ->
case Ash.Type.apply_constraints(
field_constraints[:type],
field_value,
field_constraints[:constraints] || []
) do
{:ok, nil} ->
if field_constraints[:allow_nil?] == false do
{:error, [[message: "value must not be nil", field: field]]}
else
{:ok, Map.put(result, field, nil)}
end
{:ok, field_value} ->
{:ok, Map.put(result, field, field_value)}
{:error, errors} ->
{:error, Enum.map(errors, fn error -> Keyword.put(error, :field, field) end)}
end
:error ->
{:error, [[message: "invalid value", field: field]]}
end
end
defp fetch_field(map, atom) when is_atom(atom) do
case Map.fetch(map, atom) do
{:ok, value} -> {:ok, value}
:error -> fetch_field(map, to_string(atom))
end
end
defp fetch_field(map, key), do: Map.fetch(map, key)
end

View file

@ -53,6 +53,12 @@ defmodule Mix.Tasks.Ash.Install do
:"Ash.Domain",
@domain_default_section_order
)
|> Igniter.Project.Config.configure(
"config.exs",
:ash,
[:include_embedded_source_by_default?],
true
)
|> then(fn igniter ->
if "--example" in argv do
generate_example(igniter, argv)

View file

@ -989,19 +989,23 @@ defmodule Ash.Test.Actions.BulkCreateTest do
|> Ash.create!()
assert [_] =
[%{title: "title1", authorize?: true}, %{title: "title2", authorize?: true}]
[
%{title: "title1", authorize?: true},
%{title: "title2", authorize?: true},
%{title: "title3", authorize?: true}
]
|> Ash.bulk_create!(
Post,
:create_with_policy,
tenant: org.id,
authorize?: true,
batch_size: 1,
batch_size: 2,
return_records?: true,
return_stream?: true
)
|> Enum.take(1)
assert Ash.count!(Post, authorize?: false) == 1
assert Ash.count!(Post, authorize?: false) == 2
end
test "by returning notifications, you get the notifications in the stream" do

188
test/type/struct_test.exs Normal file
View file

@ -0,0 +1,188 @@
defmodule Type.StructTest do
use ExUnit.Case, async: true
alias Ash.Test.Domain, as: Domain
defmodule Metadata do
defstruct [:foo, :bar]
end
defmodule Post do
@moduledoc false
use Ash.Resource, domain: Domain, data_layer: Ash.DataLayer.Ets
ets do
private?(true)
end
actions do
default_accept :*
defaults [:read, :destroy, create: :*, update: :*]
end
attributes do
uuid_primary_key :id
attribute :metadata, :struct do
public? true
constraints instance_of: Metadata,
fields: [
foo: [type: :string, allow_nil?: false],
bar: [type: :integer, constraints: [min: 0]]
]
end
end
end
test "it handles valid maps" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{
foo: "bar",
bar: 1
}
})
assert changeset.valid?
end
test "allow_nil? is true by default" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{
foo: "bar",
bar: "2"
}
})
assert changeset.valid?
assert changeset.attributes == %{
metadata: %Metadata{foo: "bar", bar: 2}
}
end
test "cast result has only atom keys" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{
"bar" => nil,
foo: "bar"
}
})
assert changeset.valid?
assert changeset.attributes == %{
metadata: %Metadata{foo: "bar", bar: nil}
}
end
test "keys that can be nil don't need to be there" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{
foo: "bar"
}
})
assert changeset.valid?
end
test "keys that can not be nil need to be there" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{bar: 1}
})
refute changeset.valid?
assert [
%Ash.Error.Changes.InvalidAttribute{
field: :foo,
message: "field must be present",
private_vars: nil,
value: %{bar: 1},
bread_crumbs: [],
vars: [],
path: [:metadata]
}
] = changeset.errors
end
test "constraints of field types are checked" do
changeset =
Post
|> Ash.Changeset.for_create(:create, %{
metadata: %{foo: "hello", bar: -1}
})
refute changeset.valid?
assert [
%Ash.Error.Changes.InvalidAttribute{
field: :bar,
message: "must be more than or equal to %{min}",
private_vars: nil,
value: %{bar: -1, foo: "hello"},
bread_crumbs: [],
vars: [min: 0],
path: [:metadata]
}
] = changeset.errors
end
test "extra fields are removed" do
changeset =
Post
|> Ash.Changeset.for_create(
:create,
%{
metadata: %{
"foo" => "bar",
extra: "field"
}
}
)
assert changeset.valid?
assert changeset.attributes == %{
metadata: %Metadata{foo: "bar"}
}
end
test "values are casted before checked" do
changeset =
Post
|> Ash.Changeset.for_create(
:create,
%{
metadata: %{
"foo" => "",
bar: "2"
}
}
)
refute changeset.valid?
assert [
%Ash.Error.Changes.InvalidAttribute{
field: :foo,
message: "value must not be nil",
private_vars: nil,
value: %{:bar => "2", "foo" => ""},
bread_crumbs: [],
vars: [],
path: [:metadata]
}
] = changeset.errors
end
end