From 3cd04758ec64f1dced64dcd474acd6d5ab56550c Mon Sep 17 00:00:00 2001 From: Peter Hartman Date: Mon, 13 Nov 2023 18:21:38 +0000 Subject: [PATCH] improvement: Add custom pagination start() callback for initial page request (#8) --- .gitignore | 3 + .vscode/settings.json | 2 - lib/data_layer/data_layer.ex | 33 +++++++---- lib/paginator/continuation_property.ex | 4 ++ lib/paginator/paginator.ex | 3 + mix.exs | 3 +- mix.lock | 1 + test/custom_pagination_test.exs | 82 +++++++++++++++++++++++--- test/test_helper.exs | 1 + 9 files changed, 107 insertions(+), 25 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index f6c181f..29510f4 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ ash_json_api_wrapper-*.tar # OS X folder metadata .DS_Store + +# VSCode settings +.vscode/* \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 7a73a41..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/lib/data_layer/data_layer.ex b/lib/data_layer/data_layer.ex index b019df0..03e097f 100644 --- a/lib/data_layer/data_layer.ex +++ b/lib/data_layer/data_layer.ex @@ -617,20 +617,27 @@ defmodule AshJsonApiWrapper.DataLayer do Logger.warning( "ash_json_api_wrapper does not support limits with offsets yet, and so they will both be applied after." ) - - query - else - case endpoint.limit_with do - {:param, param} -> - %{ - query - | query_params: Map.put(query.query_params || %{}, param, query.limit) - } - - _ -> - query - end end + + case endpoint.limit_with do + {:param, param} -> + %{ + query + | query_params: Map.put(query.query_params || %{}, param, query.limit) + } + + _ -> + query + end + else + query + end + + query = + if endpoint.paginator do + %{paginator: {module, opts}} = endpoint + {:ok, instructions} = module.start(opts) + apply_instructions(query, instructions) else query end diff --git a/lib/paginator/continuation_property.ex b/lib/paginator/continuation_property.ex index 9dcad4d..a257f13 100644 --- a/lib/paginator/continuation_property.ex +++ b/lib/paginator/continuation_property.ex @@ -2,6 +2,10 @@ defmodule AshJsonApiWrapper.Paginator.ContinuationProperty do @moduledoc "A paginator that uses a continuation property to paginate" use AshJsonApiWrapper.Paginator + def start(_opts) do + {:ok, %{}} + end + def continue(_response, [], _), do: :halt def continue(response, _entities, opts) do diff --git a/lib/paginator/paginator.ex b/lib/paginator/paginator.ex index 74bce6a..3f2df37 100644 --- a/lib/paginator/paginator.ex +++ b/lib/paginator/paginator.ex @@ -11,6 +11,9 @@ defmodule AshJsonApiWrapper.Paginator do end end + @callback start(opts :: Keyword.t()) :: + {:ok, %{optional(:params) => map, optional(:headers) => map}} + @callback continue( response :: term, entities :: [Ash.Resource.record()], diff --git a/mix.exs b/mix.exs index 4c4eae7..886df87 100644 --- a/mix.exs +++ b/mix.exs @@ -145,7 +145,8 @@ defmodule AshJsonApiWrapper.MixProject do {:git_ops, "~> 2.5", only: :dev}, {:excoveralls, "~> 0.13.0", only: [:dev, :test]}, {:mix_test_watch, "~> 1.0", only: :dev, runtime: false}, - {:parse_trans, "3.3.0", only: [:dev, :test], override: true} + {:parse_trans, "3.3.0", only: [:dev, :test], override: true}, + {:mox, "~> 1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index 18b618e..2fc9855 100644 --- a/mix.lock +++ b/mix.lock @@ -29,6 +29,7 @@ "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mix_test_watch": {:hex, :mix_test_watch, "1.1.1", "eee6fc570d77ad6851c7bc08de420a47fd1e449ef5ccfa6a77ef68b72e7e51ad", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "f82262b54dee533467021723892e15c3267349849f1f737526523ecba4e6baae"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, diff --git a/test/custom_pagination_test.exs b/test/custom_pagination_test.exs index 8c44512..a888273 100644 --- a/test/custom_pagination_test.exs +++ b/test/custom_pagination_test.exs @@ -1,14 +1,36 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do use ExUnit.Case require Ash.Query + import Mox @moduletag :custom_pagination + # Make sure mocks are verified when the test exits + setup :verify_on_exit! + + defmodule TestingTesla do + use Tesla + + adapter(AshJsonApiWrapper.MockAdapter) + # plug(Tesla.Middleware.Logger) + + plug(Tesla.Middleware.Retry, + delay: 2000, + max_retries: 5, + max_delay: 4_000, + should_retry: fn + {:ok, %{status: status}} when status in [429] -> true + {:ok, _} -> false + {:error, _} -> true + end + ) + end + # ── Custom paginator ── defmodule CustomPaginator do use AshJsonApiWrapper.Paginator - def cursor do + defp cursor do case :ets.whereis(:cursor) do :undefined -> :ets.new(:cursor, [:set, :protected, :named_table]) @@ -22,22 +44,27 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do end end - def increment_cursor do + defp increment_cursor do :ets.insert(:cursor, {self(), cursor() + 1}) end - def reset_cursor do + defp reset_cursor do + cursor() :ets.insert(:cursor, {self(), 1}) end - def continue(_response, [], _) do + def start(_opts) do reset_cursor() + {:ok, %{params: %{"p" => 1}}} + end + + def continue(_response, [], _) do :halt end def continue(_response, _entities, _opts) do increment_cursor() - {:ok, %{params: %{:p => cursor()}}} + {:ok, %{params: %{"p" => cursor()}}} end end @@ -49,7 +76,7 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do validate_api_inclusion?: false json_api_wrapper do - tesla(Tesla) + tesla(TestingTesla) endpoints do base("https://65383945a543859d1bb1528e.mockapi.io/api/v1") @@ -99,13 +126,50 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do Application.put_env(:ash, :validate_api_resource_inclusion?, false) Application.put_env(:ash, :validate_api_config_inclusion?, false) + AshJsonApiWrapper.MockAdapter + |> expect(:call, 4, fn env, _options -> + case env.query do + %{"l" => 3, "p" => 2} -> + {:ok, %Tesla.Env{env | status: 200, body: "[]"}} + + %{"l" => 3, "p" => 1} -> + {:ok, + %Tesla.Env{ + env + | status: 200, + body: """ + [ + {"name": "Kendra Ernser", "id":"1"}, + {"name": "Max Hartman", "id":"2"}, + {"name": "John Benton", "id":"3"} + ] + """ + }} + + query -> + {:ok, + %Tesla.Env{ + env + | status: 500, + body: "Unexpected parameters: #{query |> Kernel.inspect()}" + }} + end + end) + users = Users |> Ash.Query.for_read(:list_users) - |> Api.read!(page: [limit: 99]) + # |> Ash.Query.limit(2) + |> Api.read!(page: [limit: 2, offset: 0]) - user_count = users.results |> Enum.count() + users2 = + Users + |> Ash.Query.for_read(:list_users) + |> Api.read!(page: [limit: 2, offset: 1]) - assert(user_count == 99) + users_count = users.results |> Enum.count() + users2_count = users2.results |> Enum.count() + + assert(users_count == users2_count) end end diff --git a/test/test_helper.exs b/test/test_helper.exs index e778860..bd174f4 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,3 @@ ExUnit.start() +Mox.defmock(AshJsonApiWrapper.MockAdapter, for: Tesla.Adapter) ExUnit.configure(exclude: [:hackernews])