From 4a37edd47eee1345c6388a1482ba84a3e89f6290 Mon Sep 17 00:00:00 2001 From: Zach Daniel Date: Thu, 12 Oct 2023 18:08:34 -0400 Subject: [PATCH] improvement: rewrite package search and storage to use sqlite improvement: clean up a bunch of unused code --- .formatter.exs | 2 +- .gitignore | 1 + Dockerfile | 9 +- config/config.exs | 6 +- config/dev.exs | 6 + config/prod.exs | 6 + config/runtime.exs | 3 - config/test.exs | 4 + fly.toml | 7 +- lib/ash_hq/accounts/resources/user/user.ex | 4 - lib/ash_hq/application.ex | 3 + lib/ash_hq/ashley/ashley.ex | 12 - lib/ash_hq/ashley/http_client.ex | 51 --- lib/ash_hq/ashley/open_ai.ex | 54 --- lib/ash_hq/ashley/pinecone.ex | 14 - lib/ash_hq/ashley/registry.ex | 10 - .../resources/conversation/conversation.ex | 94 ----- .../ashley/resources/question/actions/ask.ex | 192 ---------- .../question/changes/validate_limit.ex | 23 -- .../ashley/resources/question/question.ex | 154 -------- .../ashley/resources/question/types/source.ex | 34 -- .../ashley/workers/index_library_version.ex | 233 ------------ lib/ash_hq/discord/discord.ex | 8 - lib/ash_hq/discord/listener.ex | 136 +------ lib/ash_hq/discord/poller.ex | 183 --------- lib/ash_hq/discord/registry.ex | 15 - lib/ash_hq/discord/resources/attachment.ex | 35 -- lib/ash_hq/discord/resources/channel.ex | 43 --- lib/ash_hq/discord/resources/message.ex | 96 ----- lib/ash_hq/discord/resources/reaction.ex | 43 --- lib/ash_hq/discord/resources/tag.ex | 45 --- lib/ash_hq/discord/resources/thread.ex | 109 ------ lib/ash_hq/discord/resources/thread_tag.ex | 39 -- lib/ash_hq/docs/extensions/search/index.ex | 1 + .../search/preparations/load_search_data.ex | 21 -- lib/ash_hq/docs/extensions/search/search.ex | 9 - .../transformers/add_search_structure.ex | 282 +------------- lib/ash_hq/docs/extensions/search/types.ex | 2 +- .../docs/extensions/search/types/ts_vector.ex | 11 - lib/ash_hq/docs/importer/importer.ex | 43 +-- lib/ash_hq/docs/indexer.ex | 214 +++++++++++ lib/ash_hq/docs/resources/dsl/dsl.ex | 84 ++--- .../docs/resources/extension/extension.ex | 38 +- .../docs/resources/function/function.ex | 74 ++-- lib/ash_hq/docs/resources/guide/guide.ex | 80 ++-- lib/ash_hq/docs/resources/library/agent.ex | 2 - lib/ash_hq/docs/resources/library/library.ex | 28 +- .../library_version/library_version.ex | 37 +- .../sort_by_sortable_version_instead.ex | 16 - .../docs/resources/mix_task/mix_task.ex | 57 +-- lib/ash_hq/docs/resources/module/module.ex | 64 ++-- lib/ash_hq/docs/resources/option/option.ex | 82 ++-- lib/ash_hq/release.ex | 10 +- lib/ash_hq/sqlite_repo.ex | 4 + lib/ash_hq_web/components/search.ex | 169 ++++----- lib/ash_hq_web/pages/ashley.ex | 353 ------------------ lib/ash_hq_web/views/app_view_live.ex | 6 +- litefs.yml | 53 +++ mix.exs | 9 +- mix.lock | 19 +- .../20231012101707_migrate_resources58.exs | 32 ++ .../repo/discord_messages/20231012101707.json | 95 +++++ .../repo/dsls/20220330010100.json | 2 +- .../repo/dsls/20220401004558.json | 2 +- .../repo/dsls/20220714162036.json | 2 +- .../repo/dsls/20220725201015.json | 2 +- .../repo/dsls/20220801201027.json | 2 +- .../repo/dsls/20220801201109.json | 2 +- .../repo/dsls/20220801231200.json | 2 +- .../repo/dsls/20220914230709.json | 2 +- .../repo/dsls/20221031143406.json | 2 +- .../repo/dsls/20230115062845.json | 2 +- .../repo/dsls/20230117205917.json | 2 +- .../repo/dsls/20230201064634.json | 2 +- .../repo/dsls/20230201153040.json | 2 +- .../repo/dsls/20230511170543.json | 2 +- .../repo/dsls/20230924205459.json | 2 +- .../repo/dsls/20230927050655.json | 2 +- .../repo/extensions/20220330010100.json | 2 +- .../repo/extensions/20220401004558.json | 2 +- .../repo/extensions/20220725201015.json | 2 +- .../repo/extensions/20220801201027.json | 2 +- .../repo/extensions/20220801201109.json | 2 +- .../repo/extensions/20220914230709.json | 2 +- .../repo/extensions/20221031143406.json | 2 +- .../repo/extensions/20230201064634.json | 2 +- .../repo/extensions/20230201153040.json | 2 +- .../repo/extensions/20230206143259.json | 2 +- .../repo/functions/20220330010100.json | 2 +- .../repo/functions/20220330013252.json | 2 +- .../repo/functions/20220401004558.json | 2 +- .../repo/functions/20220604165434.json | 2 +- .../repo/functions/20220714162036.json | 2 +- .../repo/functions/20220725201015.json | 2 +- .../repo/functions/20220801201027.json | 2 +- .../repo/functions/20220801201109.json | 2 +- .../repo/functions/20220914230709.json | 2 +- .../repo/functions/20221031143406.json | 2 +- .../repo/functions/20230201064634.json | 2 +- .../repo/functions/20230201153040.json | 2 +- .../repo/functions/20230216071704.json | 2 +- .../repo/functions/20230924205459.json | 2 +- .../repo/guides/20220330010100.json | 2 +- .../repo/guides/20220401004558.json | 2 +- .../repo/guides/20220405055516.json | 2 +- .../repo/guides/20220406215157.json | 2 +- .../repo/guides/20220714162036.json | 2 +- .../repo/guides/20220725201015.json | 2 +- .../repo/guides/20220801201027.json | 2 +- .../repo/guides/20220801201109.json | 2 +- .../repo/guides/20220806220239.json | 2 +- .../repo/guides/20220914230709.json | 2 +- .../repo/guides/20221031143406.json | 2 +- .../repo/guides/20221103061339.json | 2 +- .../repo/guides/20221103150903.json | 2 +- .../repo/guides/20221215044418.json | 2 +- .../repo/guides/20221215082555.json | 2 +- .../repo/guides/20230201064634.json | 2 +- .../repo/guides/20230201153040.json | 2 +- .../repo/library_versions/20220330010100.json | 2 +- .../repo/library_versions/20220604165435.json | 2 +- .../repo/library_versions/20220714164602.json | 2 +- .../repo/library_versions/20220714173908.json | 2 +- .../repo/library_versions/20220714174649.json | 2 +- .../repo/library_versions/20220725201015.json | 2 +- .../repo/library_versions/20220801201027.json | 2 +- .../repo/library_versions/20220801201109.json | 2 +- .../repo/library_versions/20220816042933.json | 2 +- .../repo/library_versions/20220823190931.json | 2 +- .../repo/library_versions/20220914230709.json | 2 +- .../repo/library_versions/20221031143406.json | 2 +- .../repo/library_versions/20230117145347.json | 2 +- .../repo/library_versions/20230201064634.json | 2 +- .../repo/mix_tasks/20220929022009.json | 2 +- .../repo/mix_tasks/20221031143406.json | 2 +- .../repo/mix_tasks/20230201064634.json | 2 +- .../repo/mix_tasks/20230201153040.json | 2 +- .../repo/mix_tasks/20230926203118.json | 2 +- .../repo/modules/20220330010100.json | 2 +- .../repo/modules/20220330013252.json | 2 +- .../repo/modules/20220401004558.json | 2 +- .../repo/modules/20220605161424.json | 2 +- .../repo/modules/20220605162941.json | 2 +- .../repo/modules/20220714162036.json | 2 +- .../repo/modules/20220725201015.json | 2 +- .../repo/modules/20220801201027.json | 2 +- .../repo/modules/20220801201109.json | 2 +- .../repo/modules/20220825205417.json | 2 +- .../repo/modules/20220914230709.json | 2 +- .../repo/modules/20221031143406.json | 2 +- .../repo/modules/20230201064634.json | 2 +- .../repo/modules/20230201153040.json | 2 +- .../repo/options/20220330010100.json | 2 +- .../repo/options/20220401004558.json | 2 +- .../repo/options/20220714162036.json | 2 +- .../repo/options/20220725201015.json | 2 +- .../repo/options/20220801201027.json | 2 +- .../repo/options/20220801201109.json | 2 +- .../repo/options/20220801231200.json | 2 +- .../repo/options/20220914230709.json | 2 +- .../repo/options/20221031143406.json | 2 +- .../repo/options/20230201064634.json | 2 +- .../repo/options/20230201153040.json | 2 +- .../repo/options/20230924205459.json | 2 +- .../repo/options/20230927050655.json | 2 +- .../repo/users/20230401210721.json | 2 +- .../sqlite_repo/dsls/20231012101705.json | 277 ++++++++++++++ .../extensions/20231012101705.json | 172 +++++++++ .../sqlite_repo/functions/20231012101705.json | 223 +++++++++++ .../sqlite_repo/guides/20231012101705.json | 162 ++++++++ .../sqlite_repo/libraries/20231012101705.json | 150 ++++++++ .../library_versions/20231012101705.json | 112 ++++++ .../sqlite_repo/mix_tasks/20231012101705.json | 152 ++++++++ .../sqlite_repo/modules/20231012101705.json | 142 +++++++ .../sqlite_repo/options/20231012101705.json | 235 ++++++++++++ priv/scripts/build_dsl_docs.exs | 33 +- .../20231012101705_migrate_resources1.exs | 304 +++++++++++++++ scripts/deploy | 1 - scripts/migrate | 3 + scripts/start | 3 + 180 files changed, 2886 insertions(+), 2860 deletions(-) delete mode 100644 lib/ash_hq/ashley/ashley.ex delete mode 100644 lib/ash_hq/ashley/http_client.ex delete mode 100644 lib/ash_hq/ashley/open_ai.ex delete mode 100644 lib/ash_hq/ashley/pinecone.ex delete mode 100644 lib/ash_hq/ashley/registry.ex delete mode 100644 lib/ash_hq/ashley/resources/conversation/conversation.ex delete mode 100644 lib/ash_hq/ashley/resources/question/actions/ask.ex delete mode 100644 lib/ash_hq/ashley/resources/question/changes/validate_limit.ex delete mode 100644 lib/ash_hq/ashley/resources/question/question.ex delete mode 100644 lib/ash_hq/ashley/resources/question/types/source.ex delete mode 100644 lib/ash_hq/ashley/workers/index_library_version.ex delete mode 100644 lib/ash_hq/discord/discord.ex delete mode 100644 lib/ash_hq/discord/poller.ex delete mode 100644 lib/ash_hq/discord/registry.ex delete mode 100644 lib/ash_hq/discord/resources/attachment.ex delete mode 100644 lib/ash_hq/discord/resources/channel.ex delete mode 100644 lib/ash_hq/discord/resources/message.ex delete mode 100644 lib/ash_hq/discord/resources/reaction.ex delete mode 100644 lib/ash_hq/discord/resources/tag.ex delete mode 100644 lib/ash_hq/discord/resources/thread.ex delete mode 100644 lib/ash_hq/discord/resources/thread_tag.ex create mode 100644 lib/ash_hq/docs/extensions/search/index.ex delete mode 100644 lib/ash_hq/docs/extensions/search/preparations/load_search_data.ex delete mode 100644 lib/ash_hq/docs/extensions/search/types/ts_vector.ex create mode 100644 lib/ash_hq/docs/indexer.ex delete mode 100644 lib/ash_hq/docs/resources/library_version/preparations/sort_by_sortable_version_instead.ex create mode 100644 lib/ash_hq/sqlite_repo.ex delete mode 100644 lib/ash_hq_web/pages/ashley.ex create mode 100644 litefs.yml create mode 100644 priv/repo/migrations/20231012101707_migrate_resources58.exs create mode 100644 priv/resource_snapshots/repo/discord_messages/20231012101707.json create mode 100644 priv/resource_snapshots/sqlite_repo/dsls/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/extensions/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/functions/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/guides/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/libraries/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/library_versions/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/mix_tasks/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/modules/20231012101705.json create mode 100644 priv/resource_snapshots/sqlite_repo/options/20231012101705.json create mode 100644 priv/sqlite_repo/migrations/20231012101705_migrate_resources1.exs create mode 100755 scripts/migrate create mode 100755 scripts/start diff --git a/.formatter.exs b/.formatter.exs index a489856..e9569f1 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -4,6 +4,7 @@ :phoenix, :ash, :ash_postgres, + :ash_sqlite, :ash_graphql, :surface, :ash_admin, @@ -23,7 +24,6 @@ locals_without_parens: [ has_name_attribute?: 1, name_attribute: 1, - library_version_attribute: 1, load_for_search: 1, doc_attribute: 1, render_attributes: 1, diff --git a/.gitignore b/.gitignore index 3d4f3ba..1002fbf 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ npm-debug.log /assets/node_modules/ .elixir_ls +ash-hq.db* /indexes/* diff --git a/Dockerfile b/Dockerfile index c15a5d4..05bc844 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM hexpm/elixir:1.15.4-erlang-26.0.2-ubuntu-bionic-20230126 +FROM hexpm/elixir:1.15.4-erlang-26.0.2-ubuntu-focal-20230126 # install build dependencies USER root RUN apt-get update @@ -13,9 +13,13 @@ RUN apt-get install -y g++ RUN apt-get install -y make RUN apt-get install -y curl RUN apt-get install -y build-essential +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get install -y esl-erlang RUN apt-get install -y apt-transport-https RUN apt-get install -y ca-certificates +RUN apt-get install -y fuse3 libfuse3-dev libglib2.0-dev +RUN apt-get install -y sqlite3 +COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs ENV NODE_MAJOR=16 RUN mkdir -p /etc/apt/keyrings RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg @@ -45,4 +49,5 @@ COPY ./config/runtime.exs config/runtime.exs COPY ./rel ./rel RUN mix release --overwrite RUN mkdir indexes -CMD ["_build/prod/rel/ash_hq/bin/ash_hq", "start"] +COPY ./litefs.yml ./litefs.yml +ENTRYPOINT litefs mount diff --git a/config/config.exs b/config/config.exs index 2664ccf..69d5db0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -8,7 +8,7 @@ import Config config :ash_hq, - ecto_repos: [AshHq.Repo] + ecto_repos: [AshHq.Repo, AshHq.SqliteRepo] config :ash, allow_flow: true @@ -27,9 +27,7 @@ config :appsignal, :config, revision: "test-4" config :ash_hq, ash_apis: [ AshHq.Accounts, - AshHq.Ashley, AshHq.Blog, - AshHq.Discord, AshHq.Docs, AshHq.Github, AshHq.MailingList @@ -88,8 +86,6 @@ config :esbuild, env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)} ] -config :open_ai, :http_client_impl, AshHq.Ashley.HttpClient - # Configures Elixir's Logger config :logger, :console, format: "$time $metadata[$level] $message\n", diff --git a/config/dev.exs b/config/dev.exs index a2aea26..5483e4c 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -22,6 +22,12 @@ config :git_ops, manage_readme_version: "README.md", version_tag_prefix: "v" +config :ash_hq, AshHq.SqliteRepo, + database: Path.join(__DIR__, "../ash-hq.db"), + port: 5432, + show_sensitive_data_on_connection_error: true, + pool_size: 10 + config :ash_hq, :show_search_ranking, true secret_key_base = "FxKFwVYhDFah3bLLXXqWdpdcLf5e5T1UyVM6XQp7kCt/Reg5yuAEI3upAVDRoP5e" diff --git a/config/prod.exs b/config/prod.exs index e643467..d9ded01 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,6 +20,12 @@ config :ash_hq, :analytics?, true config :ash_hq, :download_ua_on_start, true +if config_env() == :prod do + config :ash_hq, AshHq.SqliteRepo, + database: "/litefs/db", + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10") +end + # ## SSL Support # # To get SSL working, you will need to add the `https` key diff --git a/config/runtime.exs b/config/runtime.exs index 94aa22b..aa4f448 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -12,9 +12,6 @@ if System.get_env("PHX_SERVER") && System.get_env("RELEASE_NAME") do config :ash_hq, AshHqWeb.Endpoint, server: true end -config :open_ai, - api_key: System.get_env("OPEN_API_API_KEY") - config :ash_hq, :github, api_key: System.get_env("GITHUB_API_KEY"), client_id: System.get_env("GITHUB_CLIENT_ID"), diff --git a/config/test.exs b/config/test.exs index 2823ce3..7d7b1cc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -24,6 +24,10 @@ config :ash_hq, AshHqWeb.Endpoint, secret_key_base: secret_key_base, server: false +config :ash_hq, AshHq.SqliteRepo, + database: Path.join(__DIR__, "../ash-hq#{System.get_env("MIX_TEST_PARTITION")}.db"), + pool_size: 10 + config :ash_hq, cloak_key: "J6ED3yBWjlaOW/5byrukZTEryKa++yXWblJuhP91Qq8=" # In test we don't send emails. diff --git a/fly.toml b/fly.toml index b7afa77..2fc3f97 100644 --- a/fly.toml +++ b/fly.toml @@ -6,12 +6,13 @@ kill_signal = "SIGINT" kill_timeout = 5 processes = [] -[deploy] - release_command = "_build/prod/rel/ash_hq/bin/ash_hq eval 'AshHq.Release.migrate'" - [env] RELEASE_COOKIE = "VsipafjUVIYVpiYiljPg6DNZB8XiSnEf4zLi8WOf9bAU0XK7HuHQqA==" +[mounts] +source = "litefs" +destination = "/var/lib/litefs" + [[services]] internal_port = 4000 protocol = "tcp" diff --git a/lib/ash_hq/accounts/resources/user/user.ex b/lib/ash_hq/accounts/resources/user/user.ex index 092aac4..e0b487c 100644 --- a/lib/ash_hq/accounts/resources/user/user.ex +++ b/lib/ash_hq/accounts/resources/user/user.ex @@ -192,10 +192,6 @@ defmodule AshHq.Accounts.User do attribute :shirt_size, :string attribute :github_info, :map - attribute :ashley_access, :boolean do - default false - end - create_timestamp :created_at update_timestamp :updated_at end diff --git a/lib/ash_hq/application.ex b/lib/ash_hq/application.ex index 695cee0..de9aafc 100644 --- a/lib/ash_hq/application.ex +++ b/lib/ash_hq/application.ex @@ -34,6 +34,7 @@ defmodule AshHq.Application do AshHq.Vault, # Start the Ecto repository AshHq.Repo, + AshHq.SqliteRepo, # Start the Telemetry supervisor AshHqWeb.Telemetry, # Start the PubSub system @@ -42,6 +43,8 @@ defmodule AshHq.Application do AshHqWeb.Endpoint, {AshHq.Docs.Library.Agent, nil}, {Cluster.Supervisor, [topologies, [name: AshHq.ClusterSupervisor]]}, + {Haystack.Storage.ETS, storage: AshHq.Docs.Indexer.storage()}, + AshHq.Docs.Indexer, AshHq.Github.Monitor # Start a worker by calling: AshHq.Worker.start_link(arg) # {AshHq.Worker, arg} diff --git a/lib/ash_hq/ashley/ashley.ex b/lib/ash_hq/ashley/ashley.ex deleted file mode 100644 index 555b90e..0000000 --- a/lib/ash_hq/ashley/ashley.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule AshHq.Ashley do - @moduledoc false - use Ash.Api - - resources do - registry(AshHq.Ashley.Registry) - end - - authorization do - authorize(:by_default) - end -end diff --git a/lib/ash_hq/ashley/http_client.ex b/lib/ash_hq/ashley/http_client.ex deleted file mode 100644 index e0da7d1..0000000 --- a/lib/ash_hq/ashley/http_client.ex +++ /dev/null @@ -1,51 +0,0 @@ -defmodule AshHq.Ashley.HttpClient do - @moduledoc false - - alias OpenAI.Behaviours.HttpClientBehaviour - alias OpenAI.Error - - @behaviour HttpClientBehaviour - - @impl HttpClientBehaviour - - def request(_, _, _, %{stream: true}, _) do - {:error, - %Error{ - message: "Streaming server-sent events is not currently supported by this client." - }} - end - - def request(method, url, headers, params, opts) do - case do_request(method, url, headers, params, opts) do - {:ok, %Finch.Response{body: body}} -> {:ok, body} - {:error, error} -> {:error, error} - end - end - - @impl HttpClientBehaviour - def multipart_request(:post, url, headers, multipart, opts) do - body_stream = Multipart.body_stream(multipart) - content_type = Multipart.content_type(multipart, "multipart/form-data") - content_length = Multipart.content_length(multipart) - - headers = [ - {"Content-Type", content_type}, - {"Content-Length", to_string(content_length)} | headers - ] - - request(:post, url, headers, {:stream, body_stream}, opts) - end - - defp do_request(method, url, headers, nil, opts) do - Finch.build(method, url, headers, nil, opts) |> Finch.request(OpenAI.Finch) - end - - defp do_request(:post, url, headers, {:stream, _} = params, opts) do - Finch.build(:post, url, headers, params, opts) |> Finch.request(OpenAI.Finch) - end - - defp do_request(method, url, headers, params, opts) do - Finch.build(method, url, headers, Jason.encode!(params), opts) - |> Finch.request(OpenAI.Finch, receive_timeout: :infinity) - end -end diff --git a/lib/ash_hq/ashley/open_ai.ex b/lib/ash_hq/ashley/open_ai.ex deleted file mode 100644 index 9b7772a..0000000 --- a/lib/ash_hq/ashley/open_ai.ex +++ /dev/null @@ -1,54 +0,0 @@ -defmodule AshHq.Ashley.OpenAi do - @moduledoc false - @open_ai_embed_model "text-embedding-ada-002" - @open_ai_chat_model "gpt-4" - @message_token_limit 7000 - - @dialyzer {:nowarn_function, {:complete, 4}} - - def create_embeddings(embeddings) do - OpenAI.Embeddings.create(@open_ai_embed_model, embeddings, user: "ash-hq-importer") - end - - def complete(system_message, system_message_tokens, messages, user_email) do - OpenAI.Chat.create_completion( - @open_ai_chat_model, - [ - %{role: :system, content: system_message} - | fit_to_tokens(messages, @message_token_limit - system_message_tokens) - ], - user: user_email, - temperature: 0.2 - ) - end - - defp fit_to_tokens(messages, remaining) do - messages - |> Enum.reverse() - |> Enum.reduce_while({remaining, []}, fn message, {remaining, stack} -> - tokens = tokens(message) - - if tokens <= remaining do - {:cont, {remaining - tokens, [message | stack]}} - else - {:halt, {remaining, stack}} - end - end) - |> elem(1) - end - - def tokens(%{content: message}) do - tokens(message) - end - - def tokens(message) do - # Can't link to my python, so I'm just making a conservative estimate - # a token is ~4 chars - # Tiktoken.CL100K.encode_ordinary(message) - - message - |> String.length() - |> div(3) - |> Kernel.+(4) - end -end diff --git a/lib/ash_hq/ashley/pinecone.ex b/lib/ash_hq/ashley/pinecone.ex deleted file mode 100644 index e14f8da..0000000 --- a/lib/ash_hq/ashley/pinecone.ex +++ /dev/null @@ -1,14 +0,0 @@ -defmodule AshHq.Ashley.Pinecone do - @moduledoc false - @pinecone_opts [ - environment: "eu-west1-gcp", - project: "ba28bca", - index: "ash-hq-docs" - ] - - def client do - Pinecone.Client.new( - Keyword.put_new(@pinecone_opts, :api_key, System.get_env("PINECONE_API_KEY")) - ) - end -end diff --git a/lib/ash_hq/ashley/registry.ex b/lib/ash_hq/ashley/registry.ex deleted file mode 100644 index a7d85af..0000000 --- a/lib/ash_hq/ashley/registry.ex +++ /dev/null @@ -1,10 +0,0 @@ -defmodule AshHq.Ashley.Registry do - @moduledoc false - use Ash.Registry, - extensions: [Ash.Registry.ResourceValidations] - - entries do - entry AshHq.Ashley.Question - entry AshHq.Ashley.Conversation - end -end diff --git a/lib/ash_hq/ashley/resources/conversation/conversation.ex b/lib/ash_hq/ashley/resources/conversation/conversation.ex deleted file mode 100644 index c34e7fb..0000000 --- a/lib/ash_hq/ashley/resources/conversation/conversation.ex +++ /dev/null @@ -1,94 +0,0 @@ -defmodule AshHq.Ashley.Conversation do - @moduledoc false - use Ash.Resource, - data_layer: AshPostgres.DataLayer, - authorizers: [Ash.Policy.Authorizer] - - @conversation_limit 10 - - def conversation_limit, do: @conversation_limit - - actions do - defaults [:read, :destroy] - - create :create do - accept [:name] - end - - update :update do - accept [:name] - end - - update :ask do - transaction? false - - argument :question, :string do - allow_nil? false - end - - manual fn changeset, _ -> - changeset.data - |> AshHq.Ashley.load!(:over_limit) - |> Map.get(:over_limit) - |> if do - Ash.Changeset.add_error(%{changeset | data: %{changeset.data | over_limit: true}}, - message: "Conversation limit reached", - field: :question - ) - else - AshHq.Ashley.Question.ask!(changeset.argument.question, changeset.data.id) - end - end - end - end - - attributes do - uuid_primary_key :id - attribute :name, :string - end - - relationships do - has_many :questions, AshHq.Ashley.Question - - belongs_to :user, AshHq.Accounts.User do - api AshHq.Accounts - allow_nil? false - end - end - - policies do - policy action_type(:create) do - authorize_if relating_to_actor(:user) - end - - policy action_type([:read, :update, :destroy]) do - authorize_if relates_to_actor_via(:user) - end - end - - postgres do - table "conversations" - repo AshHq.Repo - end - - code_interface do - define_for AshHq.Ashley - define :create, args: [:name] - define :read - define :destroy - end - - changes do - change relate_actor(:user), on: [:create] - end - - aggregates do - count :question_count, [:questions] do - filter expr(success) - end - end - - calculations do - calculate :over_limit, :boolean, expr(question_count > ^@conversation_limit) - end -end diff --git a/lib/ash_hq/ashley/resources/question/actions/ask.ex b/lib/ash_hq/ashley/resources/question/actions/ask.ex deleted file mode 100644 index 429dd9c..0000000 --- a/lib/ash_hq/ashley/resources/question/actions/ask.ex +++ /dev/null @@ -1,192 +0,0 @@ -defmodule AshHq.Ashley.Question.Actions.Ask do - @moduledoc false - use Ash.Resource.ManualCreate - - require Logger - - @dialyzer {:nowarn_function, {:create, 3}} - @dialyzer {:nowarn_function, {:sources, 1}} - - @system_message_limit 4000 - - @static_context """ - You are an assistant for helping users find relevant documentation about the Ash Framework for the programming language Elixir. - Above all else, you should provide links to relevant documentation. If you don’t know the answer, do not make things up, and instead say, “Sorry, I’m not sure about that.” - - Use the following context from our documentation for your answer. All answers should be based on the documentation provided only. - - Example Resource: - defmodule Post do - use Ash.Resource - - actions do - defaults [:read, :update, :destroy] - - create :create do - accept [:text] - change {Slugify, field: text} - end - end - - attributes do - uuid_primary_key :id - attribute :text, :string, allow_nil?: false - attribute :slug, :string, allow_nil?: false - end - - relationships do - belongs_to :author, User - end - end - """ - - @system_message_tokens AshHq.Ashley.OpenAi.tokens(@static_context) - - def create(changeset, _, %{actor: actor}) do - question = Ash.Changeset.get_attribute(changeset, :question) - - {:ok, %{"data" => [%{"embedding" => vector} | _]}} = - AshHq.Ashley.OpenAi.create_embeddings([question]) - - {prompt, sources, system_message, system_message_tokens} = - AshHq.Ashley.Pinecone.client() - |> Pinecone.Vector.query(%{ - vector: vector, - topK: 10, - includeMetadata: true, - includeValues: true - }) - |> case do - {:ok, - %{ - "matches" => [] - }} -> - { - %{ - role: :user, - content: question - }, - [], - @static_context, - @system_message_tokens - } - - {:ok, - %{ - "matches" => matches - }} -> - # This is inefficient - context = - Enum.map_join( - matches, - "\n", - &""" - #{&1["metadata"]["name"]}: - #{&1["metadata"]["text"]} - """ - ) - - system_message = - """ - #{@static_context} - - #{context} - """ - |> String.slice(0..@system_message_limit) - - { - %{ - role: :user, - content: question - }, - matches, - system_message, - AshHq.Ashley.OpenAi.tokens(system_message) - } - end - - conversation_id = Ash.Changeset.get_attribute(changeset, :conversation_id) - - conversation_messages = - AshHq.Ashley.Question.history!( - actor.id, - conversation_id, - query: Ash.Query.select(AshHq.Ashley.Question, [:question, :answer]), - actor: actor - ) - |> Enum.flat_map(fn message -> - [ - %{ - role: :user, - content: message.question - }, - %{ - role: :assistant, - content: message.answer - } - ] - end) - - case AshHq.Ashley.OpenAi.complete( - system_message, - system_message_tokens, - conversation_messages ++ [prompt], - actor.email - ) do - {:ok, - %{ - "choices" => [ - %{ - "message" => %{ - "content" => answer - } - } - | _ - ] - }} -> - answer = """ - #{answer} - """ - - AshHq.Ashley.Question.create( - conversation_id, - actor.id, - question, - answer, - true, - sources(sources), - authorize?: false - ) - - {:error, error} -> - Logger.error(""" - Something went wrong creating a completion - - #{Exception.message(error)} - """) - - AshHq.Ashley.Question.create( - conversation_id, - actor.id, - question, - "Something went wrong", - false, - sources(sources), - authorize?: false - ) - end - end - - defp sources([]), do: "" - - defp sources(sources) do - sources - |> Enum.map( - &%{ - link: &1["metadata"]["link"], - name: &1["metadata"]["name"] - } - ) - |> Enum.filter(& &1) - end -end diff --git a/lib/ash_hq/ashley/resources/question/changes/validate_limit.ex b/lib/ash_hq/ashley/resources/question/changes/validate_limit.ex deleted file mode 100644 index bd80ded..0000000 --- a/lib/ash_hq/ashley/resources/question/changes/validate_limit.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule AshHq.Ashley.Question.Changes.ValidateLimit do - @moduledoc false - use Ash.Resource.Change - - def change(changeset, opts, %{actor: actor}) do - Ash.Changeset.before_action( - changeset, - fn changeset -> - count = - AshHq.Ashley.Question - |> Ash.Query.for_read(:questions_in_time_frame, %{}, actor: actor) - |> AshHq.Ashley.count!() - - if count >= opts[:question_limit] do - Ash.Changeset.add_error(changeset, message: "Question Quota Reached", field: :question) - else - changeset - end - end, - prepend?: true - ) - end -end diff --git a/lib/ash_hq/ashley/resources/question/question.ex b/lib/ash_hq/ashley/resources/question/question.ex deleted file mode 100644 index 40cc8ae..0000000 --- a/lib/ash_hq/ashley/resources/question/question.ex +++ /dev/null @@ -1,154 +0,0 @@ -defmodule AshHq.Ashley.Question do - @moduledoc false - use Ash.Resource, - data_layer: AshPostgres.DataLayer, - authorizers: [Ash.Policy.Authorizer], - extensions: [AshHq.Docs.Extensions.RenderMarkdown] - - @time_frame_hours 24 - @question_limit 10 - - actions do - defaults [:read, :destroy] - - read :history do - argument :user, :uuid do - allow_nil? false - end - - argument :conversation, :uuid do - allow_nil? false - end - - prepare build(sort: [inserted_at: :asc]) - - filter expr(user_id == ^arg(:user) and conversation_id == ^arg(:conversation)) - end - - create :ask do - transaction? false - accept [:question, :conversation_id] - allow_nil_input [:conversation_id] - - argument :conversation_name, :string - - change fn changeset, %{actor: actor} -> - Ash.Changeset.before_action(changeset, fn changeset -> - if Ash.Changeset.get_attribute(changeset, :conversation_id) do - changeset - else - conversation = - AshHq.Ashley.Conversation.create!( - changeset.arguments[:conversation_name] || "New Conversation", - actor: actor - ) - - Ash.Changeset.force_change_attribute(changeset, :conversation_id, conversation.id) - end - end) - end - - change fn changeset, _ -> - Ash.Changeset.timeout(changeset, :timer.minutes(3)) - end - - change {AshHq.Ashley.Question.Changes.ValidateLimit, limit: @question_limit} - - manual AshHq.Ashley.Question.Actions.Ask - end - - create :create do - accept [:conversation_id, :question, :answer, :success, :user_id, :sources] - end - - read :questions_in_time_frame do - filter expr( - user_id == ^actor(:id) and inserted_at >= ago(@time_frame_hours, :hour) and success - ) - end - end - - attributes do - uuid_primary_key :id - - attribute :question, :string do - allow_nil? false - end - - attribute :answer, :string do - allow_nil? false - end - - attribute :sources, {:array, AshHq.Ashley.Question.Types.Source} do - allow_nil? false - default [] - end - - attribute :answer_html, :string do - allow_nil? false - end - - attribute :success, :boolean do - allow_nil? false - end - - timestamps() - end - - render_markdown do - render_attributes answer: :answer_html - end - - relationships do - belongs_to :user, AshHq.Accounts.User do - allow_nil? false - api AshHq.Accounts - attribute_writable? true - end - - belongs_to :conversation, AshHq.Ashley.Conversation do - allow_nil? false - attribute_writable? true - end - end - - policies do - policy always() do - authorize_if actor_present() - end - - policy action_type(:read) do - authorize_if action(:history) - authorize_if accessing_from(AshHq.Ashley.Conversation, :questions) - authorize_if expr(user_id == ^actor(:id)) - end - - policy action(:history) do - authorize_if expr(^actor(:id) == ^arg(:user)) - end - - policy action(:create) do - forbid_if always() - end - end - - postgres do - table "questions" - repo AshHq.Repo - - migration_defaults sources: "[]" - - references do - reference :conversation, on_delete: :delete - end - end - - code_interface do - define_for AshHq.Ashley - - define :questions_in_time_frame - define :ask, args: [:question] - define :create, args: [:conversation_id, :user_id, :question, :answer, :success, :sources] - define :history, args: [:user, :conversation] - end -end diff --git a/lib/ash_hq/ashley/resources/question/types/source.ex b/lib/ash_hq/ashley/resources/question/types/source.ex deleted file mode 100644 index dbb0de4..0000000 --- a/lib/ash_hq/ashley/resources/question/types/source.ex +++ /dev/null @@ -1,34 +0,0 @@ -defmodule AshHq.Ashley.Question.Types.Source do - @moduledoc false - use Ash.Resource, - data_layer: :embedded - - actions do - create :create do - primary? true - allow_nil_input [:name] - - change fn changeset, _ -> - if Ash.Changeset.get_attribute(changeset, :name) do - changeset - else - Ash.Changeset.force_change_attribute( - changeset, - :name, - Ash.Changeset.get_attribute(changeset, :link) - ) - end - end - end - end - - attributes do - attribute :link, :string do - allow_nil? false - end - - attribute :name, :string do - allow_nil? false - end - end -end diff --git a/lib/ash_hq/ashley/workers/index_library_version.ex b/lib/ash_hq/ashley/workers/index_library_version.ex deleted file mode 100644 index fba0c85..0000000 --- a/lib/ash_hq/ashley/workers/index_library_version.ex +++ /dev/null @@ -1,233 +0,0 @@ -defmodule AshHq.Ashley.Workers.IndexLibraryVersion do - @moduledoc false - require Ash.Query - - @dialyzer {:nowarn_function, {:delete_vectors, 2}} - @dialyzer {:nowarn_function, {:index_all, 0}} - @dialyzer {:nowarn_function, {:perform, 1}} - @dialyzer {:nowarn_function, {:guides, 1}} - @dialyzer {:nowarn_function, {:modules, 1}} - @dialyzer {:nowarn_function, {:functions, 1}} - @dialyzer {:nowarn_function, {:dsls, 1}} - @dialyzer {:nowarn_function, {:options, 1}} - @dialyzer {:nowarn_function, {:name, 1}} - @dialyzer {:nowarn_function, {:path, 2}} - @dialyzer {:nowarn_function, {:format, 2}} - - def index_all do - AshHq.Docs.Library.read!(load: :latest_version_id) - |> Enum.filter(& &1.latest_version_id) - |> Enum.each(&perform(&1.latest_version_id)) - end - - def perform(id) do - pinecone_client = AshHq.Ashley.Pinecone.client() - - library_version = AshHq.Docs.get!(AshHq.Docs.LibraryVersion, id, load: :library) - delete_vectors(pinecone_client, library_version) - - guides(library_version) - |> Stream.concat(modules(library_version)) - |> Stream.concat(functions(library_version)) - |> Stream.concat(dsls(library_version)) - |> Stream.concat(options(library_version)) - |> Stream.map(fn item -> - {item, format(item, library_version)} - end) - |> Stream.chunk_every(100) - |> Stream.map(fn batch -> - case AshHq.Ashley.OpenAi.create_embeddings(Enum.map(batch, &elem(&1, 1))) do - {:ok, %{"data" => data}} -> - vectors = - Enum.zip_with(data, batch, fn %{"embedding" => values}, {item, text} -> - %{ - values: values, - id: item.id, - metadata: %{ - library: library_version.library.name, - link: "#{path(item, library_version.library.name)}", - name: "#{name(item)}", - text: text - } - } - end) - - Pinecone.Vector.upsert(pinecone_client, %{vectors: vectors}) - - {:error, error} -> - {:error, error} - end - end) - |> Stream.run() - end - - defp guides(library_version) do - AshHq.Docs.Guide - |> Ash.Query.filter(library_version_id == ^library_version.id) - |> AshHq.Docs.stream() - end - - defp modules(library_version) do - AshHq.Docs.Module - |> Ash.Query.filter(library_version_id == ^library_version.id) - |> AshHq.Docs.stream() - end - - defp functions(library_version) do - AshHq.Docs.Function - |> Ash.Query.filter(library_version_id == ^library_version.id) - |> Ash.Query.load(:module_name) - |> AshHq.Docs.stream() - end - - defp dsls(library_version) do - AshHq.Docs.Dsl - |> Ash.Query.filter(library_version_id == ^library_version.id) - |> Ash.Query.load(:extension_target) - |> AshHq.Docs.stream() - end - - defp options(library_version) do - AshHq.Docs.Option - |> Ash.Query.filter(library_version_id == ^library_version.id) - |> Ash.Query.load(:extension_target) - |> AshHq.Docs.stream() - end - - defp name(%AshHq.Docs.Option{} = option) do - case option.path do - [] -> - "#{option.extension_target} - #{option.name}" - - path -> - "#{option.extension_target} - #{Enum.join(path, ".")} | #{option.name}" - end - end - - defp name(%AshHq.Docs.Dsl{} = dsl) do - case dsl.path do - [] -> - "#{dsl.extension_target} - #{dsl.name}" - - path -> - "#{dsl.extension_target} - #{Enum.join(path ++ [dsl.name], ".")}" - end - end - - defp name(%AshHq.Docs.Function{} = function) do - "#{function.type} - #{function.module_name}.#{function.name}/#{function.arity}" - end - - defp name(%AshHq.Docs.Module{} = module) do - module.name - end - - defp name(%AshHq.Docs.Guide{} = guide) do - guide.name - end - - defp path(%AshHq.Docs.Option{} = option, _library_name) do - "docs/dsl/#{sanitize_name(option.extension_target)}##{String.replace(option.sanitized_path, "/", "-")}-#{sanitize_name(option.name)}" - end - - defp path(%AshHq.Docs.Dsl{} = option, _library_name) do - "docs/dsl/#{sanitize_name(option.extension_target)}##{String.replace(option.sanitized_path, "/", "-")}" - end - - defp path( - %AshHq.Docs.Function{ - sanitized_name: sanitized_name, - arity: arity, - type: type, - module_name: module_name - }, - library_name - ) do - "/docs/module/#{library_name}/latest/#{sanitize_name(module_name)}##{type}-#{sanitized_name}-#{arity}" - end - - defp path( - %AshHq.Docs.Module{ - sanitized_name: sanitized_name - }, - library_name - ) do - "/docs/module/#{library_name}/latest/#{sanitized_name}" - end - - defp path( - %AshHq.Docs.Guide{ - route: route - }, - library_name - ) do - "/docs/guides/#{library_name}/latest/#{route}" - end - - defp format(%AshHq.Docs.Option{} = option, _library_version) do - """ - DSL Entity: #{Enum.join(option.path ++ [option.name])} - Type: #{option.type} - Default: #{option.default} - #{option.doc} - """ - end - - defp format(%AshHq.Docs.Dsl{type: :entity} = dsl, _library_version) do - """ - DSL Entity: #{Enum.join(dsl.path ++ [dsl.name])} - #{Enum.map_join(dsl.examples || [], &"```\n#{&1}\n```")} - #{dsl.doc} - """ - end - - defp format(%AshHq.Docs.Dsl{type: :section} = dsl, _library_version) do - """ - DSL Section: #{Enum.join(dsl.path ++ [dsl.name])} - #{Enum.map_join(dsl.examples || [], &"```\n#{&1}\n```")} - #{dsl.doc} - """ - end - - defp format(%AshHq.Docs.Function{} = function, _library_version) do - """ - #{function.type} #{function.module_name}.#{function.name}/#{function.arity} - Types: - #{Enum.join(function.heads, "\n")} - - Docs: - #{function.doc} - """ - end - - defp format(%AshHq.Docs.Module{} = module, _library_version) do - """ - #{module.name}: - #{module.doc} - """ - end - - defp format(%AshHq.Docs.Guide{} = guide, _library_version) do - """ - #{guide.name}: - #{guide.text} - """ - end - - def sanitize_name(name, allow_forward_slash? \\ false) do - if allow_forward_slash? do - String.downcase(String.replace(to_string(name), ~r/[^A-Za-z0-9\/_]/, "-")) - else - String.downcase(String.replace(to_string(name), ~r/[^A-Za-z0-9_]/, "-")) - end - end - - defp delete_vectors(pinecone_client, library_version) do - pinecone_client - |> Pinecone.Vector.delete(%{ - filter: %{ - library: library_version.library.name - } - }) - end -end diff --git a/lib/ash_hq/discord/discord.ex b/lib/ash_hq/discord/discord.ex deleted file mode 100644 index fbdd386..0000000 --- a/lib/ash_hq/discord/discord.ex +++ /dev/null @@ -1,8 +0,0 @@ -defmodule AshHq.Discord do - @moduledoc "Discord api import & interactions" - use Ash.Api - - resources do - registry(AshHq.Discord.Registry) - end -end diff --git a/lib/ash_hq/discord/listener.ex b/lib/ash_hq/discord/listener.ex index 3a84534..b0cf1b0 100644 --- a/lib/ash_hq/discord/listener.ex +++ b/lib/ash_hq/discord/listener.ex @@ -5,99 +5,33 @@ defmodule AshHq.Discord.Listener do use Nostrum.Consumer import Bitwise - @all_types AshHq.Docs.Extensions.Search.Types.types() -- ["Forum"] @user_id 1_066_406_803_769_933_834 + @server_id 711_271_361_523_351_632 def start_link() do Consumer.start_link(__MODULE__) end def search_results!(interaction) do - search = - interaction.data.options - |> Enum.find_value(fn option -> - if option.name == "search" do - option.value - end - end) + item_list = AshHq.Docs.Indexer.search!(search) - type = - interaction.data.options - |> Enum.find_value(fn option -> - if option.name == "type" do - option.value - end - end) + item_list = Enum.take(item_list, 10) - library = - interaction.data.options - |> Enum.find_value(fn option -> - if option.name == "library" do - option.value - end - end) + count = + case Enum.count(item_list) do + 10 -> + "the top 10" - libraries = - AshHq.Docs.Library.read!() - |> Enum.filter(& &1.latest_version_id) - - library_version_ids = - if library do - case Enum.find(libraries, &(&1.name == library)) do - nil -> - [] - - library -> - [library.latest_version_id] - end - else - Enum.map(libraries, & &1.latest_version_id) + other -> + "#{other}" end - input = - if type do - %{types: [type]} - else - %{types: @all_types} - end + """ + Found #{count} results for "#{search}": - %{result: item_list} = AshHq.Docs.Search.run!(search, library_version_ids, input) - - result_type = - if type do - "#{type} results" - else - "results" - end - - library = - if library do - "#{library}" - else - "all libraries" - end - - if item_list do - item_list = Enum.take(item_list, 10) - - count = - case Enum.count(item_list) do - 10 -> - "the top 10" - - other -> - "#{other}" - end - - """ - Found #{count} #{result_type} in #{library} for query "#{search}": - - #{Enum.map_join(item_list, "\n", &render_search_result(&1))} - """ - else - "Something went wrong." - end + #{Enum.map_join(item_list, "\n", &render_search_result(&1))} + """ end defp render_search_result(item) do @@ -161,19 +95,11 @@ defmodule AshHq.Discord.Listener do def rebuild do if Application.get_env(:ash_hq, :discord_bot) do - libraries = - AshHq.Docs.Library.read!() - |> Enum.filter(& &1.latest_library_version) - - build_search_action(libraries) + build_search_action() end end - defp build_search_action(libraries) do - library_names = - libraries - |> Enum.map(& &1.name) - + defp build_search_action() do command = %{ name: "ash_hq_search", description: "Search AshHq Documentation", @@ -185,36 +111,6 @@ defmodule AshHq.Discord.Listener do description: "what you want to search for", required: true }, - %{ - # ApplicationCommandType::STRING - type: 3, - name: "type", - description: "What type of thing you want to search for. Defaults to everything.", - required: false, - choices: - Enum.map(@all_types, fn type -> - %{ - name: String.downcase(type), - description: "Search only for #{String.downcase(type)} items.", - value: type - } - end) - }, - %{ - # ApplicationCommandType::STRING - type: 3, - name: "library", - description: "Which library you'd like to search. Defaults to all libraries.", - required: false, - choices: - Enum.map(library_names, fn name -> - %{ - name: name, - description: "Search only in the #{name} library.", - value: name - } - end) - }, %{ # ApplicationCommandType::Boolean type: 5, @@ -227,7 +123,7 @@ defmodule AshHq.Discord.Listener do Nostrum.Api.create_guild_application_command( @user_id, - AshHq.Discord.Poller.server_id(), + @server_id, command ) end diff --git a/lib/ash_hq/discord/poller.ex b/lib/ash_hq/discord/poller.ex deleted file mode 100644 index dc80ff2..0000000 --- a/lib/ash_hq/discord/poller.ex +++ /dev/null @@ -1,183 +0,0 @@ -defmodule AshHq.Discord.Poller do - @moduledoc """ - Every 2 hours, synchronizes all active threads and the 50 most recent archived threads - """ - - use GenServer - require Logger - - @poll_interval :timer.hours(1) - @server_id 711_271_361_523_351_632 - @archived_thread_lookback 50 - - @channels [ - 1_066_222_835_758_014_606, - 1_066_223_107_922_210_867, - 1_019_647_368_196_534_283 - ] - - def server_id, do: @server_id - - defmacrop unwrap(value) do - quote do - case unquote(value) do - {:ok, value} -> - value - - {:error, error} -> - raise Exception.format(:error, error, []) - end - end - end - - def start_link(state) do - GenServer.start_link(__MODULE__, state, name: __MODULE__) - end - - def init(_) do - send(self(), :poll) - {:ok, nil} - end - - def handle_info(:poll, state) do - poll() - Process.send_after(self(), :poll, @poll_interval) - {:noreply, state} - end - - def poll do - for {channel, index} <- Enum.with_index(@channels) do - channel - |> Nostrum.Api.get_channel!() - |> tap(fn channel -> - channel - |> Map.from_struct() - |> Map.put(:order, index) - |> AshHq.Discord.Channel.upsert!() - end) - |> Map.get(:available_tags) - |> Enum.each(fn available_tag -> - AshHq.Discord.Tag.upsert!(channel, available_tag.id, available_tag.name) - end) - end - - active = - @server_id - |> Nostrum.Api.list_guild_threads() - |> unwrap() - |> Map.get(:threads) - |> Stream.filter(fn thread -> - thread.parent_id in @channels - end) - |> Stream.map(fn thread -> - %{ - thread: thread, - messages: get_all_channel_messages(thread.id) - } - end) - - archived = - @channels - |> Stream.flat_map(fn channel -> - channel - |> Nostrum.Api.list_public_archived_threads(limit: @archived_thread_lookback) - |> unwrap() - |> Map.get(:threads) - |> Enum.map(fn thread -> - messages = - thread.id - |> get_all_channel_messages() - - %{ - thread: thread, - messages: messages - } - end) - end) - - active - |> Stream.concat(archived) - |> Enum.reject(fn - %{messages: []} -> - true - - _ -> - false - end) - |> Enum.each(fn %{thread: thread, messages: messages} -> - try do - author = - messages - |> Enum.min_by(& &1.timestamp, DateTime) - |> Map.get(:author) - - thread - |> Map.put(:author, author) - |> Map.from_struct() - |> Map.put(:channel_id, thread.parent_id) - |> Map.put(:tags, thread.applied_tags) - |> Map.put(:create_timestamp, thread.thread_metadata.create_timestamp) - |> Map.put(:messages, Enum.map(messages, &Map.from_struct/1)) - |> AshHq.Discord.Thread.upsert!() - rescue - e -> - Logger.error( - "Failed to import message:\n #{Exception.format(:error, e, __STACKTRACE__)}" - ) - end - end) - end - - defp get_all_channel_messages(thread) do - Stream.resource( - fn -> - :all - end, - fn - nil -> - {:halt, nil} - - before -> - locator = - case before do - :all -> - nil - - before -> - {:before, before} - end - - messages = - if locator do - Nostrum.Api.get_channel_messages!(thread, 100, locator) - else - Nostrum.Api.get_channel_messages!(thread, 100) - end - - if Enum.count(messages) == 100 do - {messages, List.last(messages).id} - else - {messages, nil} - end - end, - & &1 - ) - |> Stream.map(fn message -> - message - |> Map.put(:author, message.author.username) - |> Map.update!(:reactions, fn reactions -> - reactions - |> Kernel.||([]) - # just don't know what this looks like, so removing them - |> Enum.reject(&(is_nil(&1.emoji) || &1.emoji == "" || &1.emoji.animated)) - |> Enum.map(fn %{count: count, emoji: emoji} -> - %{emoji: emoji.name, count: count} - end) - end) - |> Map.update!(:attachments, fn attachments -> - Enum.map(attachments, &Map.from_struct/1) - end) - end) - |> Enum.to_list() - end -end diff --git a/lib/ash_hq/discord/registry.ex b/lib/ash_hq/discord/registry.ex deleted file mode 100644 index b92b050..0000000 --- a/lib/ash_hq/discord/registry.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule AshHq.Discord.Registry do - @moduledoc false - use Ash.Registry, - extensions: [Ash.Registry.ResourceValidations] - - entries do - entry AshHq.Discord.Attachment - entry AshHq.Discord.Channel - entry AshHq.Discord.Message - entry AshHq.Discord.Reaction - entry AshHq.Discord.Tag - entry AshHq.Discord.Thread - entry AshHq.Discord.ThreadTag - end -end diff --git a/lib/ash_hq/discord/resources/attachment.ex b/lib/ash_hq/discord/resources/attachment.ex deleted file mode 100644 index 54d124f..0000000 --- a/lib/ash_hq/discord/resources/attachment.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule AshHq.Discord.Attachment do - @moduledoc "A discord attachment on a message" - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - actions do - defaults [:create, :read, :update, :destroy] - end - - attributes do - integer_primary_key :id, generated?: false, writable?: true - attribute :filename, :string - attribute :size, :integer - attribute :url, :string - attribute :proxy_url, :string - attribute :height, :integer - attribute :width, :integer - end - - relationships do - belongs_to :message, AshHq.Discord.Message do - allow_nil? false - attribute_type :integer - end - end - - postgres do - table "discord_attachments" - repo AshHq.Repo - - references do - reference :message, on_delete: :delete, on_update: :update - end - end -end diff --git a/lib/ash_hq/discord/resources/channel.ex b/lib/ash_hq/discord/resources/channel.ex deleted file mode 100644 index a9725bd..0000000 --- a/lib/ash_hq/discord/resources/channel.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule AshHq.Discord.Channel do - @moduledoc """ - The channel is the discord forum channel. We explicitly configure which ones we import. - """ - - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - actions do - defaults [:create, :read, :update, :destroy] - - create :upsert do - upsert? true - end - end - - attributes do - integer_primary_key :id, writable?: true, generated?: false - - attribute :name, :string do - allow_nil? false - end - - attribute :order, :integer do - allow_nil? false - end - end - - relationships do - has_many :threads, AshHq.Discord.Thread - end - - postgres do - table "discord_channels" - repo AshHq.Repo - end - - code_interface do - define_for AshHq.Discord - define :read - define :upsert - end -end diff --git a/lib/ash_hq/discord/resources/message.ex b/lib/ash_hq/discord/resources/message.ex deleted file mode 100644 index 26d6eb7..0000000 --- a/lib/ash_hq/discord/resources/message.ex +++ /dev/null @@ -1,96 +0,0 @@ -defmodule AshHq.Discord.Message do - @moduledoc """ - Discord messages synchronized by the discord bot - """ - use Ash.Resource, - data_layer: AshPostgres.DataLayer, - extensions: [ - AshHq.Docs.Extensions.RenderMarkdown, - AshHq.Docs.Extensions.Search - ] - - actions do - defaults [:read, :destroy] - - create :create do - primary? true - argument :attachments, {:array, :map} - argument :reactions, {:array, :map} - change manage_relationship(:attachments, type: :direct_control) - - change manage_relationship(:reactions, - type: :direct_control, - use_identities: [:unique_message_emoji] - ) - end - - update :update do - primary? true - argument :attachments, {:array, :map} - argument :reactions, {:array, :map} - change manage_relationship(:attachments, type: :direct_control) - - change manage_relationship(:reactions, - type: :direct_control, - use_identities: [:unique_message_emoji] - ) - end - end - - render_markdown do - render_attributes content: :content_html - end - - search do - doc_attribute :content - - type "Forum" - - load_for_search [ - :channel_name, - :thread_name - ] - - has_name_attribute? false - weight_content(-0.7) - end - - attributes do - integer_primary_key :id, generated?: false, writable?: true - - attribute :author, :string do - allow_nil? false - end - - attribute :content, :string - attribute :content_html, :string - - attribute :timestamp, :utc_datetime do - allow_nil? false - end - end - - relationships do - belongs_to :thread, AshHq.Discord.Thread do - attribute_type :integer - allow_nil? false - end - - has_many :attachments, AshHq.Discord.Attachment - has_many :reactions, AshHq.Discord.Reaction - end - - postgres do - table "discord_messages" - repo AshHq.Repo - - references do - reference :thread, on_delete: :delete, on_update: :update - end - end - - aggregates do - first :channel_name, [:thread, :channel], :name - first :thread_name, [:thread], :name - end -end diff --git a/lib/ash_hq/discord/resources/reaction.ex b/lib/ash_hq/discord/resources/reaction.ex deleted file mode 100644 index 6a009a7..0000000 --- a/lib/ash_hq/discord/resources/reaction.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule AshHq.Discord.Reaction do - @moduledoc """ - Reactions store emoji reaction counts. - """ - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - actions do - defaults [:create, :read, :update, :destroy] - end - - attributes do - uuid_primary_key :id - - attribute :count, :integer do - allow_nil? false - end - - attribute :emoji, :string do - allow_nil? false - end - end - - relationships do - belongs_to :message, AshHq.Discord.Message do - attribute_type :integer - allow_nil? false - end - end - - postgres do - table "discord_reactions" - repo AshHq.Repo - - references do - reference :message, on_delete: :delete, on_update: :update - end - end - - identities do - identity :unique_message_emoji, [:emoji, :message_id] - end -end diff --git a/lib/ash_hq/discord/resources/tag.ex b/lib/ash_hq/discord/resources/tag.ex deleted file mode 100644 index 7386554..0000000 --- a/lib/ash_hq/discord/resources/tag.ex +++ /dev/null @@ -1,45 +0,0 @@ -defmodule AshHq.Discord.Tag do - @moduledoc "A tag that can be applied to a post. Currently uses CSV data layer and therefore is static" - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - actions do - defaults [:create, :read, :update, :destroy] - - create :upsert do - upsert? true - upsert_identity :unique_name_per_channel - end - end - - attributes do - integer_primary_key :id, generated?: false, writable?: true - - attribute :name, :ci_string do - allow_nil? false - end - end - - relationships do - belongs_to :channel, AshHq.Discord.Channel do - attribute_type :integer - attribute_writable? true - end - end - - postgres do - table "discord_tags" - repo AshHq.Repo - end - - code_interface do - define_for AshHq.Discord - define :upsert, args: [:channel_id, :id, :name] - define :read - define :destroy - end - - identities do - identity :unique_name_per_channel, [:name, :channel_id] - end -end diff --git a/lib/ash_hq/discord/resources/thread.ex b/lib/ash_hq/discord/resources/thread.ex deleted file mode 100644 index 9d277da..0000000 --- a/lib/ash_hq/discord/resources/thread.ex +++ /dev/null @@ -1,109 +0,0 @@ -defmodule AshHq.Discord.Thread do - @moduledoc """ - A thread is an individual forum post (because they are really just fancy threads) - """ - - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - import Ecto.Query - - actions do - defaults [:create, :read, :update, :destroy] - - read :feed do - pagination do - countable true - offset? true - default_limit 25 - end - - argument :channel, :integer do - allow_nil? false - end - - argument :tag_name, :string - - prepare build(sort: [create_timestamp: :desc]) - - filter expr( - channel_id == ^arg(:channel) and - (is_nil(^arg(:tag_name)) or tags.name == ^arg(:tag_name)) - ) - end - - create :upsert do - upsert? true - argument :messages, {:array, :map} - argument :tags, {:array, :integer} - - change manage_relationship(:messages, type: :direct_control) - - change fn changeset, _ -> - Ash.Changeset.after_action(changeset, fn changeset, thread -> - tags = Ash.Changeset.get_argument(changeset, :tags) || [] - - # Not optimized in `manage_relationship` - # bulk actions should make this unnecessary - to_delete = - from thread_tag in AshHq.Discord.ThreadTag, - where: thread_tag.thread_id == ^thread.id, - where: thread_tag.tag_id not in ^tags - - AshHq.Repo.delete_all(to_delete) - - Enum.map(tags, fn tag -> - AshHq.Discord.ThreadTag.tag!(thread.id, tag) - end) - - {:ok, thread} - end) - end - end - end - - attributes do - integer_primary_key :id, generated?: false, writable?: true - attribute :type, :integer - - attribute :name, :string do - allow_nil? false - end - - attribute :author, :string do - allow_nil? false - end - - attribute :create_timestamp, :utc_datetime do - allow_nil? false - end - end - - relationships do - has_many :messages, AshHq.Discord.Message - - belongs_to :channel, AshHq.Discord.Channel do - attribute_type :integer - allow_nil? false - attribute_writable? true - end - - many_to_many :tags, AshHq.Discord.Tag do - through AshHq.Discord.ThreadTag - source_attribute_on_join_resource :thread_id - destination_attribute_on_join_resource :tag_id - end - end - - postgres do - table "discord_threads" - repo AshHq.Repo - end - - code_interface do - define_for AshHq.Discord - define :upsert - define :by_id, action: :read, get_by: [:id] - define :feed, args: [:channel] - end -end diff --git a/lib/ash_hq/discord/resources/thread_tag.ex b/lib/ash_hq/discord/resources/thread_tag.ex deleted file mode 100644 index 14b9e6e..0000000 --- a/lib/ash_hq/discord/resources/thread_tag.ex +++ /dev/null @@ -1,39 +0,0 @@ -defmodule AshHq.Discord.ThreadTag do - @moduledoc "Joins a thread to a tag" - use Ash.Resource, - data_layer: AshPostgres.DataLayer - - actions do - defaults [:read, :destroy] - - create :tag do - upsert? true - end - end - - relationships do - belongs_to :thread, AshHq.Discord.Thread do - primary_key? true - allow_nil? false - attribute_writable? true - attribute_type :integer - end - - belongs_to :tag, AshHq.Discord.Tag do - primary_key? true - allow_nil? false - attribute_writable? true - attribute_type :integer - end - end - - postgres do - table "discord_thread_tags" - repo AshHq.Repo - end - - code_interface do - define_for AshHq.Discord - define :tag, args: [:thread_id, :tag_id] - end -end diff --git a/lib/ash_hq/docs/extensions/search/index.ex b/lib/ash_hq/docs/extensions/search/index.ex new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/lib/ash_hq/docs/extensions/search/index.ex @@ -0,0 +1 @@ + diff --git a/lib/ash_hq/docs/extensions/search/preparations/load_search_data.ex b/lib/ash_hq/docs/extensions/search/preparations/load_search_data.ex deleted file mode 100644 index b7e1b1d..0000000 --- a/lib/ash_hq/docs/extensions/search/preparations/load_search_data.ex +++ /dev/null @@ -1,21 +0,0 @@ -defmodule AshHq.Extensions.Search.Preparations.LoadSearchData do - @moduledoc """ - Ensures that any data needed for search results is loaded. - """ - use Ash.Resource.Preparation - - def prepare(query, _, _) do - query_string = Ash.Query.get_argument(query, :query) - to_load = AshHq.Docs.Extensions.Search.load_for_search(query.resource) - - query.resource - |> AshHq.Docs.Extensions.RenderMarkdown.render_attributes() - |> Enum.reduce(query, fn {source, target}, query -> - Ash.Query.deselect(query, [source, target]) - end) - |> Ash.Query.load(search_headline: [query: query_string]) - |> Ash.Query.load(match_rank: [query: query_string]) - |> Ash.Query.load(to_load) - |> Ash.Query.sort(match_rank: {:asc, %{query: query_string}}) - end -end diff --git a/lib/ash_hq/docs/extensions/search/search.ex b/lib/ash_hq/docs/extensions/search/search.ex index 9ad9130..88fade1 100644 --- a/lib/ash_hq/docs/extensions/search/search.ex +++ b/lib/ash_hq/docs/extensions/search/search.ex @@ -60,11 +60,6 @@ defmodule AshHq.Docs.Extensions.Search do type: :atom, doc: "The text field to be used in the search" ], - library_version_attribute: [ - type: :atom, - default: :library_version_id, - doc: "The attribute to use to filter by library version" - ], load_for_search: [ type: {:list, :any}, default: [], @@ -140,10 +135,6 @@ defmodule AshHq.Docs.Extensions.Search do |> List.wrap() end - def library_version_attribute(resource) do - Extension.get_opt(resource, [:search], :library_version_attribute, :library_version_id) - end - def load_for_search(resource) do Extension.get_opt(resource, [:search], :load_for_search, :library_version_id) end diff --git a/lib/ash_hq/docs/extensions/search/transformers/add_search_structure.ex b/lib/ash_hq/docs/extensions/search/transformers/add_search_structure.ex index c6cf674..9ea54c3 100644 --- a/lib/ash_hq/docs/extensions/search/transformers/add_search_structure.ex +++ b/lib/ash_hq/docs/extensions/search/transformers/add_search_structure.ex @@ -4,15 +4,8 @@ defmodule AshHq.Docs.Extensions.Search.Transformers.AddSearchStructure do * Adds a sanitized name attribute if it doesn't already exist * Adds a change to set the sanitized name, if it should. - * Adds a `search_headline` calculation - * Adds a `matches` calculation - * Adds relevant indexes using custom sql statements - * Adds a `match_rank` calculation. - * Adds a search action - * Adds a code interface for the search action """ use Spark.Dsl.Transformer - import Ash.Filter.TemplateHelpers require Ash.Query alias Spark.Dsl.Transformer @@ -26,36 +19,10 @@ defmodule AshHq.Docs.Extensions.Search.Transformers.AddSearchStructure do config = %{ name_attribute: name_attribute, - doc_attribute: Transformer.get_option(dsl_state, [:search], :doc_attribute), - library_version_attribute: - Transformer.get_option(dsl_state, [:search], :library_version_attribute) || - :library_version_id, - table: Transformer.get_option(dsl_state, [:postgres], :table), sanitized_name_attribute: sanitized_name_attribute } - currently_ignored_attributes = - AshPostgres.DataLayer.Info.migration_ignore_attributes(dsl_state) - - dsl_state - |> add_sanitized_name(config) - |> add_search_action(config) - |> add_code_interface() - |> Transformer.set_option([:postgres], :migration_ignore_attributes, [ - :searchable | currently_ignored_attributes - ]) - |> add_search_headline_calculation(config) - |> add_matches_calculation() - |> add_full_text_column(config) - |> add_full_text_index() - |> add_match_rank_calculation(config) - |> Ash.Resource.Builder.add_preparation( - {AshHq.Docs.Extensions.Search.Preparations.DeselectSearchable, []} - ) - |> Ash.Resource.Builder.add_attribute(:searchable, AshHq.Docs.Search.Types.TsVector, - generated?: true, - private?: true - ) + {:ok, add_sanitized_name(dsl_state, config)} end defp add_sanitized_name(dsl_state, config) do @@ -104,253 +71,6 @@ defmodule AshHq.Docs.Extensions.Search.Transformers.AddSearchStructure do end end - defp add_full_text_index(dsl_state) do - Transformer.add_entity( - dsl_state, - [:postgres, :custom_indexes], - Transformer.build_entity!( - AshPostgres.DataLayer, - [:postgres, :custom_indexes], - :index, - fields: [:searchable], - using: "GIN" - ) - ) - end - - defp add_full_text_column(dsl_state, config) do - if config.doc_attribute do - if Transformer.get_option(dsl_state, [:search], :has_name_attribute?, true) do - Transformer.add_entity( - dsl_state, - [:postgres, :custom_statements], - Transformer.build_entity!( - AshPostgres.DataLayer, - [:postgres, :custom_statements], - :statement, - name: :search_column, - up: """ - ALTER TABLE #{config.table} - ADD COLUMN searchable tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english', #{config.name_attribute}), 'A') || - setweight(to_tsvector('english', #{config.doc_attribute}), 'D') - ) STORED; - """, - down: """ - ALTER TABLE #{config.table} - DROP COLUMN searchable - """ - ) - ) - else - Transformer.add_entity( - dsl_state, - [:postgres, :custom_statements], - Transformer.build_entity!( - AshPostgres.DataLayer, - [:postgres, :custom_statements], - :statement, - name: :search_column, - up: """ - ALTER TABLE #{config.table} - ADD COLUMN searchable tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english', #{config.doc_attribute}), 'D') - ) STORED; - """, - down: """ - ALTER TABLE #{config.table} - DROP COLUMN searchable - """ - ) - ) - end - else - Transformer.add_entity( - dsl_state, - [:postgres, :custom_statements], - Transformer.build_entity!( - AshPostgres.DataLayer, - [:postgres, :custom_statements], - :statement, - name: :search_column, - up: """ - ALTER TABLE #{config.table} - ADD COLUMN searchable tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english', #{config.name_attribute}), 'A') - ) STORED; - """, - down: """ - ALTER TABLE #{config.table} - DROP COLUMN searchable - """ - ) - ) - end - end - - defp add_match_rank_calculation(dsl_state, _config) do - weight_content = AshHq.Docs.Extensions.Search.weight_content(dsl_state) - - dsl_state - |> Transformer.add_entity( - [:calculations], - Transformer.build_entity!(Ash.Resource.Dsl, [:calculations], :calculate, - name: :match_rank, - type: :float, - private?: true, - arguments: [query_argument()], - calculation: - Ash.Query.expr( - fragment( - "(ts_rank_cd('{0.05, 0.1, 0.1, 1.0}', ?, websearch_to_tsquery(?), 32) + ?)", - ^ref(:searchable), - ^arg(:query), - ^weight_content - ) - ) - ) - ) - end - - defp add_matches_calculation(dsl_state) do - Transformer.add_entity( - dsl_state, - [:calculations], - Transformer.build_entity!(Ash.Resource.Dsl, [:calculations], :calculate, - name: :matches, - type: :boolean, - private?: true, - arguments: [query_argument()], - calculation: - Ash.Query.expr( - fragment( - "(? @@ websearch_to_tsquery(?))", - ^ref(:searchable), - ^arg(:query) - ) - ) - ) - ) - end - - # defp add_name_matches_calculation(dsl_state, config) do - # if AshHq.Docs.Extensions.Search.has_name_attribute?(dsl_state) do - # Transformer.add_entity( - # dsl_state, - # [:calculations], - # Transformer.build_entity!(Ash.Resource.Dsl, [:calculations], :calculate, - # name: :name_matches, - # type: :boolean, - # arguments: [query_argument(), similarity_argument()], - # private?: true, - # calculation: - # Ash.Query.expr( - # contains( - # fragment("lower(?)", ^ref(config.name_attribute)), - # fragment("lower(?)", ^arg(:query)) - # ) - # ) - # ) - # ) - # else - # dsl_state - # end - # end - - defp add_search_headline_calculation(dsl_state, config) do - if config.doc_attribute do - Transformer.add_entity( - dsl_state, - [:calculations], - Transformer.build_entity!(Ash.Resource.Dsl, [:calculations], :calculate, - name: :search_headline, - type: :string, - private?: true, - arguments: [query_argument()], - calculation: - Ash.Query.expr( - # credo:disable-for-next-line - fragment( - "ts_headline('english', ?, websearch_to_tsquery('english', ?), 'MaxFragments=2,StartSel=\"\", StopSel=')", - ^ref(config.doc_attribute), - ^arg(:query) - ) - ) - ) - ) - else - Transformer.add_entity( - dsl_state, - [:calculations], - Transformer.build_entity!(Ash.Resource.Dsl, [:calculations], :calculate, - name: :search_headline, - type: :string, - arguments: [query_argument()], - private?: true, - calculation: Ash.Query.expr("") - ) - ) - end - end - - defp query_argument do - Transformer.build_entity!( - Ash.Resource.Dsl, - [:calculations, :calculate], - :argument, - type: :string, - name: :query, - allow_nil?: false - ) - end - - defp add_search_action(dsl_state, _config) do - query_argument = - Transformer.build_entity!( - Ash.Resource.Dsl, - [:actions, :read], - :argument, - type: :string, - name: :query - ) - - {arguments, filter} = - {[query_argument], Ash.Query.expr(matches(query: arg(:query)))} - - Transformer.add_entity( - dsl_state, - [:actions], - Transformer.build_entity!(Ash.Resource.Dsl, [:actions], :read, - name: :search, - arguments: arguments, - preparations: search_preparations(), - filter: filter - ) - ) - end - - defp add_code_interface(dsl_state) do - Transformer.add_entity( - dsl_state, - [:code_interface], - Transformer.build_entity!(Ash.Resource.Dsl, [:code_interface], :define, - name: :search, - args: [:query] - ) - ) - end - - defp search_preparations do - [ - Transformer.build_entity!(Ash.Resource.Dsl, [:actions, :read], :prepare, - preparation: AshHq.Extensions.Search.Preparations.LoadSearchData - ) - ] - end - def before?(Ash.Resource.Transformers.SetTypes), do: true def before?(_), do: false def after?(Ash.Resource.Transformers.SetPrimaryActions), do: true diff --git a/lib/ash_hq/docs/extensions/search/types.ex b/lib/ash_hq/docs/extensions/search/types.ex index 24ff789..1058c63 100644 --- a/lib/ash_hq/docs/extensions/search/types.ex +++ b/lib/ash_hq/docs/extensions/search/types.ex @@ -3,7 +3,7 @@ defmodule AshHq.Docs.Extensions.Search.Types do A static list of all search types that currently exist """ - @search_types [AshHq.Docs.Registry, AshHq.Discord.Registry] + @search_types [AshHq.Docs.Registry] |> Enum.flat_map(&Ash.Registry.Info.entries/1) |> Enum.filter(&(AshHq.Docs.Extensions.Search in Spark.extensions(&1))) |> Enum.map(&AshHq.Docs.Extensions.Search.type/1) diff --git a/lib/ash_hq/docs/extensions/search/types/ts_vector.ex b/lib/ash_hq/docs/extensions/search/types/ts_vector.ex deleted file mode 100644 index 28408b2..0000000 --- a/lib/ash_hq/docs/extensions/search/types/ts_vector.ex +++ /dev/null @@ -1,11 +0,0 @@ -defmodule AshHq.Docs.Search.Types.TsVector do - @moduledoc "A stub for a tsvector type that should never actually get loaded." - use Ash.Type - - def storage_type, do: :tsvector - def cast_in_query?(_), do: false - - defdelegate cast_input(value, constraints), to: Ash.Type.String - defdelegate cast_stored(value, constraints), to: Ash.Type.String - defdelegate dump_to_native(value, constraints), to: Ash.Type.String -end diff --git a/lib/ash_hq/docs/importer/importer.ex b/lib/ash_hq/docs/importer/importer.ex index 0e5a874..d1fdd50 100644 --- a/lib/ash_hq/docs/importer/importer.ex +++ b/lib/ash_hq/docs/importer/importer.ex @@ -135,8 +135,6 @@ defmodule AshHq.Docs.Importer do Logger.error( "Failed to import version #{name} #{version} #{Exception.format(:error, e, __STACKTRACE__)}" ) - - e end end) end @@ -219,34 +217,21 @@ defmodule AshHq.Docs.Importer do end if result do - {:ok, library_version} = - AshHq.Repo.transaction(fn -> - Logger.info("Starting import of #{name}: #{version}") + Logger.info("Starting import of #{name}: #{version}") - id = - case LibraryVersion.by_version(library.id, version) do - {:ok, version} -> - LibraryVersion.destroy!(version) - version.id - - _ -> - Ash.UUID.generate() - end - - LibraryVersion.build!( - library.id, - version, - %{ - timeout: :infinity, - id: id, - extensions: result[:extensions], - doc: result[:doc], - guides: result[:guides], - modules: result[:modules], - mix_tasks: result[:mix_tasks] - } - ) - end) + library_version = + LibraryVersion.build!( + library.id, + version, + %{ + timeout: :infinity, + extensions: result[:extensions], + doc: result[:doc], + guides: result[:guides], + modules: result[:modules], + mix_tasks: result[:mix_tasks] + } + ) LibraryVersion |> Ash.Query.for_read(:read) diff --git a/lib/ash_hq/docs/indexer.ex b/lib/ash_hq/docs/indexer.ex new file mode 100644 index 0000000..aa15c13 --- /dev/null +++ b/lib/ash_hq/docs/indexer.ex @@ -0,0 +1,214 @@ +defmodule AshHq.Docs.Indexer do + use GenServer + + require Ash.Query + + def start_link(state, opts \\ []) do + GenServer.start_link(__MODULE__, state, opts) + end + + def init(_) do + {:ok, %{haystack: haystack()}, {:continue, :index}} + end + + def haystack do + Haystack.index(Haystack.new(), :search, fn index -> + index + |> Haystack.Index.ref(Haystack.Index.Field.term("id")) + |> Haystack.Index.field(Haystack.Index.Field.new("name")) + |> Haystack.Index.field(Haystack.Index.Field.new("call_name")) + |> Haystack.Index.field(Haystack.Index.Field.new("doc")) + |> Haystack.Index.field(Haystack.Index.Field.new("library_name")) + |> Haystack.Index.storage(storage()) + end) + end + + def storage do + Haystack.Storage.ETS.new(name: :search, table: :search) + end + + def search(search) do + tokens = Haystack.Tokenizer.tokenize(search) + tokens = Haystack.Transformer.pipeline(tokens, Haystack.Transformer.default()) + + Haystack.index(haystack(), :search, fn index -> + query = + Enum.reduce(tokens, Haystack.Query.Clause.new(:any), fn token, clause -> + Enum.reduce( + Map.values(Map.take(index.fields, ["name", "call_name", "description"])), + clause, + fn field, clause -> + Haystack.Query.Clause.expressions(clause, [ + Haystack.Query.Expression.new(:match, field: field.k, term: token.v) + ]) + end + ) + end) + + Haystack.Query.new() + |> Haystack.Query.clause(query) + |> Haystack.Query.run(index) + end) + |> Stream.map(fn item -> + [type, id] = String.split(item.ref, "|") + %{id: id, type: type, score: item.score} + end) + |> Enum.group_by(& &1.type) + |> Enum.flat_map(fn {type, items} -> + resource = + case type do + "dsl" -> AshHq.Docs.Dsl + "guide" -> AshHq.Docs.Guide + "option" -> AshHq.Docs.Option + "module" -> AshHq.Docs.Module + "mix_task" -> AshHq.Docs.MixTask + "function" -> AshHq.Docs.Function + end + + ids = Enum.map(items, & &1.id) + + scores = Map.new(items, &{&1.id, &1.score}) + + resource + |> Ash.Query.filter(id in ^ids) + |> Ash.Query.load(AshHq.Docs.Extensions.Search.load_for_search(resource)) + |> AshHq.Docs.read!() + |> Enum.map(fn item -> + Ash.Resource.put_metadata(item, :search_score, scores[item.id]) + end) + end) + |> Enum.sort_by(&{!exact_match?(&1, search), -&1.__metadata__.search_score}) + end + + defp exact_match?(record, search) do + record + |> Map.take([:name, :call_name]) + |> Map.values() + |> Enum.any?(fn value -> + is_binary(value) && + String.downcase(value) == String.downcase(String.trim_trailing(search, "(")) + end) + end + + def handle_continue(:index, state) do + {:noreply, index(state)} + end + + def handle_info(:index, state) do + {:noreply, index(state)} + end + + def handle_info(_, state), do: {:noreply, state} + + defp index(state) do + haystack = + Haystack.index(state.haystack, :search, fn index -> + [ + dsls(), + guides(), + options(), + modules(), + mix_tasks(), + functions() + ] + |> Stream.concat() + |> Stream.chunk_every(100) + |> Enum.each(&Haystack.Index.add(index, &1)) + end) + + %{state | haystack: haystack} + after + Process.send_after(self(), :index, :timer.hours(6)) + end + + defp dsls do + AshHq.Docs.Dsl + |> Ash.Query.load([:library_name, :extension_module]) + |> AshHq.Docs.stream!() + |> Stream.map(fn dsl -> + %{ + "id" => id("dsl", dsl.id), + "name" => dsl.name, + "library_name" => dsl.library_name, + "doc" => dsl.doc, + "call_name" => "#{dsl.extension_module}.#{dsl.sanitized_path}.#{dsl.name}" + } + end) + end + + defp guides do + AshHq.Docs.Guide + |> Ash.Query.load(library_version: :library) + |> AshHq.Docs.stream!() + |> Stream.map(fn guide -> + %{ + "id" => id("guide", guide.id), + "name" => guide.name, + "library_name" => guide.library_version.library.name, + "doc" => guide.text + } + end) + end + + defp options do + AshHq.Docs.Option + |> Ash.Query.load([:library_name, :extension_module]) + |> AshHq.Docs.stream!() + |> Stream.map(fn option -> + %{ + "id" => id("option", option.id), + "name" => option.name, + "library_name" => option.library_name, + "doc" => option.doc, + "call_name" => "#{option.extension_module}.#{option.sanitized_path}.#{option.name}" + } + end) + end + + defp modules do + AshHq.Docs.Module + |> Ash.Query.load([:library_name]) + |> AshHq.Docs.stream!() + |> Stream.map(fn module -> + %{ + "id" => id("module", module.id), + "name" => module.name, + "library_name" => module.library_name, + "doc" => module.doc, + "call_name" => module.name + } + end) + end + + defp mix_tasks do + AshHq.Docs.MixTask + |> Ash.Query.load([:library_name]) + |> AshHq.Docs.stream!() + |> Stream.map(fn mix_task -> + %{ + "id" => id("mix_task", mix_task.id), + "name" => mix_task.module_name, + "library_name" => mix_task.library_name, + "doc" => mix_task.doc, + "call_name" => mix_task.name + } + end) + end + + defp functions do + AshHq.Docs.Function + |> Ash.Query.load([:library_name, :call_name]) + |> AshHq.Docs.stream!() + |> Stream.map(fn function -> + %{ + "id" => id("function", function.id), + "name" => function.name, + "library_name" => function.library_name, + "doc" => function.doc, + "call_name" => function.call_name + } + end) + end + + defp id(type, id), do: "#{type}|#{id}" +end diff --git a/lib/ash_hq/docs/resources/dsl/dsl.ex b/lib/ash_hq/docs/resources/dsl/dsl.ex index 256574b..2c33f8b 100644 --- a/lib/ash_hq/docs/resources/dsl/dsl.ex +++ b/lib/ash_hq/docs/resources/dsl/dsl.ex @@ -2,9 +2,41 @@ defmodule AshHq.Docs.Dsl do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "dsls" + repo AshHq.SqliteRepo + + references do + reference :library_version, on_delete: :delete + reference :dsl, on_delete: :delete + end + + migration_defaults optional_args: "[]" + end + + search do + doc_attribute :doc + + load_for_search [ + :extension_name, + :extension_target, + :extension_module, + :library_name + ] + + weight_content(0.2) + + sanitized_name_attribute :sanitized_path + use_path_for_name? true + end + + render_markdown do + render_attributes doc: :doc_html + end + actions do defaults [:update, :destroy] @@ -35,26 +67,6 @@ defmodule AshHq.Docs.Dsl do end end - search do - doc_attribute :doc - - load_for_search [ - :extension_name, - :extension_target, - :extension_module, - :library_name - ] - - weight_content(0.2) - - sanitized_name_attribute :sanitized_path - use_path_for_name? true - end - - render_markdown do - render_attributes doc: :doc_html - end - attributes do uuid_primary_key :id @@ -113,18 +125,6 @@ defmodule AshHq.Docs.Dsl do has_many :dsls, __MODULE__ end - postgres do - table "dsls" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - reference :dsl, on_delete: :delete - end - - migration_defaults optional_args: "[]" - end - code_interface do define_for AshHq.Docs define :read @@ -134,14 +134,14 @@ defmodule AshHq.Docs.Dsl do description "An entity or section in an Ash DSL" end - aggregates do - first :extension_type, :extension, :type - first :extension_order, :extension, :order - first :extension_name, :extension, :name - first :extension_module, :extension, :module - first :extension_target, :extension, :target - first :version_name, :library_version, :version - first :library_name, [:library_version, :library], :name - first :library_id, [:library_version, :library], :id + calculations do + calculate :extension_type, :string, expr(extension.type) + calculate :extension_order, :integer, expr(extension.order) + calculate :extension_name, :string, expr(extension.name) + calculate :extension_module, :string, expr(extension.module) + calculate :extension_target, :string, expr(extension.target) + calculate :version_name, :string, expr(library_version.version) + calculate :library_name, :string, expr(library_version.library.name) + calculate :library_id, :string, expr(library_version.library.id) end end diff --git a/lib/ash_hq/docs/resources/extension/extension.ex b/lib/ash_hq/docs/resources/extension/extension.ex index e535357..61df9e8 100644 --- a/lib/ash_hq/docs/resources/extension/extension.ex +++ b/lib/ash_hq/docs/resources/extension/extension.ex @@ -2,9 +2,27 @@ defmodule AshHq.Docs.Extension do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "extensions" + repo AshHq.SqliteRepo + + references do + reference :library_version, on_delete: :delete + end + end + + search do + doc_attribute :doc + load_for_search library_version: [:library_display_name, :library_name] + end + + render_markdown do + render_attributes doc: :doc_html + end + actions do defaults [:update, :destroy] @@ -31,15 +49,6 @@ defmodule AshHq.Docs.Extension do end end - search do - doc_attribute :doc - load_for_search library_version: [:library_display_name, :library_name] - end - - render_markdown do - render_attributes doc: :doc_html - end - attributes do uuid_primary_key :id @@ -86,15 +95,6 @@ defmodule AshHq.Docs.Extension do has_many :options, AshHq.Docs.Option end - postgres do - table "extensions" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs diff --git a/lib/ash_hq/docs/resources/function/function.ex b/lib/ash_hq/docs/resources/function/function.ex index a62e136..8a46544 100644 --- a/lib/ash_hq/docs/resources/function/function.ex +++ b/lib/ash_hq/docs/resources/function/function.ex @@ -2,9 +2,39 @@ defmodule AshHq.Docs.Function do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "functions" + repo AshHq.SqliteRepo + + references do + reference :library_version, on_delete: :delete + end + end + + search do + doc_attribute :doc + + load_for_search [ + :version_name, + :library_name, + :module_name, + :call_name, + :library_id + ] + + type "Code" + + show_docs_on :module_sanitized_name + end + + render_markdown do + render_attributes doc: :doc_html, heads: :heads_html + header_ids? false + end + actions do defaults [:update, :destroy] @@ -26,26 +56,6 @@ defmodule AshHq.Docs.Function do end end - search do - doc_attribute :doc - - load_for_search [ - :version_name, - :library_name, - :module_name, - :library_id - ] - - type "Code" - - show_docs_on :module_sanitized_name - end - - render_markdown do - render_attributes doc: :doc_html, heads: :heads_html - header_ids? false - end - attributes do uuid_primary_key :id @@ -105,15 +115,6 @@ defmodule AshHq.Docs.Function do end end - postgres do - table "functions" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs end @@ -122,11 +123,12 @@ defmodule AshHq.Docs.Function do description "A function in a module exposed by an Ash library" end - aggregates do - first :version_name, :library_version, :version - first :library_name, [:library_version, :library], :name - first :library_id, [:library_version, :library], :id - first :module_name, :module, :name - first :module_sanitized_name, :module, :sanitized_name + calculations do + calculate :version_name, :string, expr(library_version.version) + calculate :library_name, :string, expr(library_version.library.name) + calculate :library_id, :uuid, expr(library_version.library.id) + calculate :module_name, :string, expr(module.name) + calculate :module_sanitized_name, :string, expr(module.sanitized_name) + calculate :call_name, :string, expr(module_name <> "." <> name) end end diff --git a/lib/ash_hq/docs/resources/guide/guide.ex b/lib/ash_hq/docs/resources/guide/guide.ex index 7324920..3d36943 100644 --- a/lib/ash_hq/docs/resources/guide/guide.ex +++ b/lib/ash_hq/docs/resources/guide/guide.ex @@ -1,7 +1,7 @@ defmodule AshHq.Docs.Guide do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [ AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown, @@ -9,6 +9,43 @@ defmodule AshHq.Docs.Guide do AshAdmin.Resource ] + sqlite do + repo AshHq.SqliteRepo + table "guides" + + references do + reference :library_version, on_delete: :delete + end + end + + search do + doc_attribute :text + show_docs_on [:sanitized_name, :sanitized_route] + type "Guides" + load_for_search [:library_name, library_version: [:library_name, :library_display_name]] + end + + render_markdown do + render_attributes text: :text_html + table_of_contents? true + end + + graphql do + type :guide + + queries do + list :list_guides, :read_for_version + end + end + + admin do + form do + field :text do + type :markdown + end + end + end + actions do defaults [:create, :update, :destroy] @@ -34,34 +71,6 @@ defmodule AshHq.Docs.Guide do end end - search do - doc_attribute :text - show_docs_on [:sanitized_name, :sanitized_route] - type "Guides" - load_for_search library_version: [:library_name, :library_display_name] - end - - render_markdown do - render_attributes text: :text_html - table_of_contents? true - end - - graphql do - type :guide - - queries do - list :list_guides, :read_for_version - end - end - - admin do - form do - field :text do - type :markdown - end - end - end - attributes do uuid_primary_key :id @@ -112,15 +121,6 @@ defmodule AshHq.Docs.Guide do end end - postgres do - repo AshHq.Repo - table "guides" - - references do - reference :library_version, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs end @@ -146,4 +146,8 @@ defmodule AshHq.Docs.Guide do end end end + + calculations do + calculate :library_name, :string, expr(library_version.library.name) + end end diff --git a/lib/ash_hq/docs/resources/library/agent.ex b/lib/ash_hq/docs/resources/library/agent.ex index d41feea..07088eb 100644 --- a/lib/ash_hq/docs/resources/library/agent.ex +++ b/lib/ash_hq/docs/resources/library/agent.ex @@ -25,8 +25,6 @@ defmodule AshHq.Docs.Library.Agent do end def clear do - AshHq.Discord.Listener.rebuild() - Agent.update(__MODULE__, fn _state -> nil end) diff --git a/lib/ash_hq/docs/resources/library/library.ex b/lib/ash_hq/docs/resources/library/library.ex index 31b48f9..077de96 100644 --- a/lib/ash_hq/docs/resources/library/library.ex +++ b/lib/ash_hq/docs/resources/library/library.ex @@ -1,7 +1,14 @@ defmodule AshHq.Docs.Library do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer + data_layer: AshSqlite.DataLayer + + sqlite do + table "libraries" + repo AshHq.SqliteRepo + + migration_defaults module_prefixes: "[]", skip_versions: "[]" + end actions do defaults [:create, :update, :destroy] @@ -67,16 +74,10 @@ defmodule AshHq.Docs.Library do has_one :latest_library_version, AshHq.Docs.LibraryVersion do sort version: :desc + from_many? true end end - postgres do - table "libraries" - repo AshHq.Repo - - migration_defaults module_prefixes: "[]", skip_versions: "[]" - end - code_interface do define_for AshHq.Docs @@ -103,13 +104,8 @@ defmodule AshHq.Docs.Library do end end - aggregates do - first :latest_version, :versions, :version do - sort version: :desc - end - - first :latest_version_id, :versions, :id do - sort version: :desc - end + calculations do + calculate :latest_version, :string, expr(latest_library_version.version) + calculate :latest_version_id, :uuid, expr(latest_library_version.id) end end diff --git a/lib/ash_hq/docs/resources/library_version/library_version.ex b/lib/ash_hq/docs/resources/library_version/library_version.ex index e33da8d..fa872c7 100644 --- a/lib/ash_hq/docs/resources/library_version/library_version.ex +++ b/lib/ash_hq/docs/resources/library_version/library_version.ex @@ -2,9 +2,19 @@ defmodule AshHq.Docs.LibraryVersion do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "library_versions" + repo AshHq.SqliteRepo + end + + search do + name_attribute :version + load_for_search [:library_name, :library_display_name] + end + actions do defaults [:update, :destroy] @@ -81,12 +91,6 @@ defmodule AshHq.Docs.LibraryVersion do end end - search do - name_attribute :version - library_version_attribute :id - load_for_search [:library_name, :library_display_name] - end - attributes do uuid_primary_key :id @@ -113,11 +117,6 @@ defmodule AshHq.Docs.LibraryVersion do has_many :mix_tasks, AshHq.Docs.MixTask end - postgres do - table "library_versions" - repo AshHq.Repo - end - code_interface do define_for AshHq.Docs define :build, args: [:library, :version] @@ -143,18 +142,8 @@ defmodule AshHq.Docs.LibraryVersion do end end - preparations do - prepare AshHq.Docs.LibraryVersion.Preparations.SortBySortableVersionInstead - end - - aggregates do - first :library_name, :library, :name - first :library_display_name, :library, :display_name - end - calculations do - calculate :sortable_version, - {:array, :string}, - expr(fragment("string_to_array(?, '.')", version)) + calculate :library_name, :string, expr(library.name) + calculate :library_display_name, :string, expr(library.display_name) end end diff --git a/lib/ash_hq/docs/resources/library_version/preparations/sort_by_sortable_version_instead.ex b/lib/ash_hq/docs/resources/library_version/preparations/sort_by_sortable_version_instead.ex deleted file mode 100644 index d22199b..0000000 --- a/lib/ash_hq/docs/resources/library_version/preparations/sort_by_sortable_version_instead.ex +++ /dev/null @@ -1,16 +0,0 @@ -defmodule AshHq.Docs.LibraryVersion.Preparations.SortBySortableVersionInstead do - @moduledoc """ - Replaces any sort on `version` by a sort on `sortable_version` instead. - """ - use Ash.Resource.Preparation - - def prepare(query, _, _) do - %{query | sort: replace_sort(query.sort)} - end - - defp replace_sort(nil), do: nil - defp replace_sort(:version), do: :sortable_version - defp replace_sort({:version, order}), do: {:sortable_version, order} - defp replace_sort(list) when is_list(list), do: Enum.map(list, &replace_sort/1) - defp replace_sort(other), do: other -end diff --git a/lib/ash_hq/docs/resources/mix_task/mix_task.ex b/lib/ash_hq/docs/resources/mix_task/mix_task.ex index 151eacb..6b13e05 100644 --- a/lib/ash_hq/docs/resources/mix_task/mix_task.ex +++ b/lib/ash_hq/docs/resources/mix_task/mix_task.ex @@ -2,22 +2,15 @@ defmodule AshHq.Docs.MixTask do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] - actions do - defaults [:update, :destroy] + sqlite do + table "mix_tasks" + repo AshHq.SqliteRepo - read :read do - primary? true - pagination offset?: true, countable: true, default_limit: 25, required?: false - end - - create :create do - primary? true - argument :library_version, :uuid - - change manage_relationship(:library_version, type: :append_and_remove) + references do + reference :library_version, on_delete: :delete end end @@ -39,6 +32,27 @@ defmodule AshHq.Docs.MixTask do render_attributes doc: :doc_html end + actions do + defaults [:update, :destroy] + + read :read do + primary? true + + pagination keyset?: true, + offset?: true, + countable: true, + default_limit: 25, + required?: false + end + + create :create do + primary? true + argument :library_version, :uuid + + change manage_relationship(:library_version, type: :append_and_remove) + end + end + attributes do uuid_primary_key :id @@ -79,15 +93,6 @@ defmodule AshHq.Docs.MixTask do end end - postgres do - table "mix_tasks" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs end @@ -96,9 +101,9 @@ defmodule AshHq.Docs.MixTask do description "Represents a mix task that has been exposed by a library" end - aggregates do - first :version_name, :library_version, :version - first :library_name, [:library_version, :library], :name - first :library_id, [:library_version, :library], :id + calculations do + calculate :version_name, :string, expr(library_version.version) + calculate :library_name, :string, expr(library_version.library.name) + calculate :library_id, :uuid, expr(library_version.library.id) end end diff --git a/lib/ash_hq/docs/resources/module/module.ex b/lib/ash_hq/docs/resources/module/module.ex index 66c64c0..1cfaa35 100644 --- a/lib/ash_hq/docs/resources/module/module.ex +++ b/lib/ash_hq/docs/resources/module/module.ex @@ -2,9 +2,36 @@ defmodule AshHq.Docs.Module do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "modules" + repo AshHq.SqliteRepo + + references do + reference :library_version, on_delete: :delete + end + end + + search do + doc_attribute :doc + + weight_content(0.5) + + load_for_search [ + :version_name, + :library_name, + :library_id + ] + + type "Code" + end + + render_markdown do + render_attributes doc: :doc_html + end + actions do defaults [:update, :destroy] @@ -29,24 +56,6 @@ defmodule AshHq.Docs.Module do end end - search do - doc_attribute :doc - - weight_content(0.5) - - load_for_search [ - :version_name, - :library_name, - :library_id - ] - - type "Code" - end - - render_markdown do - render_attributes doc: :doc_html - end - attributes do uuid_primary_key :id @@ -87,15 +96,6 @@ defmodule AshHq.Docs.Module do has_many :functions, AshHq.Docs.Function end - postgres do - table "modules" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs end @@ -104,9 +104,9 @@ defmodule AshHq.Docs.Module do description "Represents a module that has been exposed by a library" end - aggregates do - first :version_name, :library_version, :version - first :library_name, [:library_version, :library], :name - first :library_id, [:library_version, :library], :id + calculations do + calculate :version_name, :string, expr(library_version.version) + calculate :library_name, :string, expr(library_version.library.name) + calculate :library_id, :uuid, expr(library_version.library.id) end end diff --git a/lib/ash_hq/docs/resources/option/option.ex b/lib/ash_hq/docs/resources/option/option.ex index 25a49af..31f2809 100644 --- a/lib/ash_hq/docs/resources/option/option.ex +++ b/lib/ash_hq/docs/resources/option/option.ex @@ -2,9 +2,39 @@ defmodule AshHq.Docs.Option do @moduledoc false use Ash.Resource, - data_layer: AshPostgres.DataLayer, + data_layer: AshSqlite.DataLayer, extensions: [AshHq.Docs.Extensions.Search, AshHq.Docs.Extensions.RenderMarkdown] + sqlite do + table "options" + repo AshHq.SqliteRepo + + references do + reference :library_version, on_delete: :delete + reference :dsl, on_delete: :delete + end + end + + search do + doc_attribute :doc + + load_for_search [ + :extension_name, + :extension_module, + :extension_target, + :library_name + ] + + sanitized_name_attribute :sanitized_path + use_path_for_name? true + add_name_to_path? false + show_docs_on :dsl_sanitized_path + end + + render_markdown do + render_attributes doc: :doc_html + end + actions do defaults [:update, :destroy] @@ -31,26 +61,6 @@ defmodule AshHq.Docs.Option do end end - search do - doc_attribute :doc - - load_for_search [ - :extension_name, - :extension_module, - :extension_target, - :library_name - ] - - sanitized_name_attribute :sanitized_path - use_path_for_name? true - add_name_to_path? false - show_docs_on :dsl_sanitized_path - end - - render_markdown do - render_attributes doc: :doc_html - end - attributes do uuid_primary_key :id @@ -105,16 +115,6 @@ defmodule AshHq.Docs.Option do end end - postgres do - table "options" - repo AshHq.Repo - - references do - reference :library_version, on_delete: :delete - reference :dsl, on_delete: :delete - end - end - code_interface do define_for AshHq.Docs define :read @@ -124,15 +124,15 @@ defmodule AshHq.Docs.Option do description "Represents an option on a DSL section or entity" end - aggregates do - first :extension_type, [:dsl, :extension], :type - first :extension_name, [:dsl, :extension], :name - first :extension_order, [:dsl, :extension], :order - first :extension_target, [:dsl, :extension], :target - first :extension_module, [:dsl, :extension], :module - first :version_name, :library_version, :version - first :library_name, [:library_version, :library], :name - first :library_id, [:library_version, :library], :id - first :dsl_sanitized_path, :dsl, :sanitized_path + calculations do + calculate :extension_type, :string, expr(dsl.extension.type) + calculate :extension_name, :string, expr(dsl.extension.name) + calculate :extension_order, :integer, expr(dsl.extension.order) + calculate :extension_target, :string, expr(dsl.extension.target) + calculate :extension_module, :string, expr(dsl.extension.module) + calculate :version_name, :string, expr(library_version.version) + calculate :library_name, :string, expr(library_version.library.name) + calculate :library_id, :string, expr(library_version.library.id) + calculate :dsl_sanitized_path, :string, expr(dsl.sanitized_path) end end diff --git a/lib/ash_hq/release.ex b/lib/ash_hq/release.ex index f62fe14..bb7b88a 100644 --- a/lib/ash_hq/release.ex +++ b/lib/ash_hq/release.ex @@ -6,6 +6,7 @@ defmodule AshHq.Release do @app :ash_hq def migrate do load_app() + System.cmd("sqlite3", ["litefs/db", "VACUUM;"]) for repo <- repos() do {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) @@ -23,14 +24,7 @@ defmodule AshHq.Release do end defp repos do - apis() - |> Enum.flat_map(fn api -> - api - |> Ash.Api.Info.resources() - |> Enum.filter(&(AshPostgres.DataLayer in Spark.extensions(&1))) - |> Enum.map(&AshPostgres.DataLayer.Info.repo/1) - end) - |> Enum.uniq() + Application.fetch_env!(@app, :ecto_repos) end defp apis do diff --git a/lib/ash_hq/sqlite_repo.ex b/lib/ash_hq/sqlite_repo.ex new file mode 100644 index 0000000..29aefa0 --- /dev/null +++ b/lib/ash_hq/sqlite_repo.ex @@ -0,0 +1,4 @@ +defmodule AshHq.SqliteRepo do + use AshSqlite.Repo, + otp_app: :ash_hq +end diff --git a/lib/ash_hq_web/components/search.ex b/lib/ash_hq_web/components/search.ex index 7024ddb..85dfaf2 100644 --- a/lib/ash_hq_web/components/search.ex +++ b/lib/ash_hq_web/components/search.ex @@ -95,6 +95,42 @@ defmodule AshHqWeb.Components.Search do ~F"""
{#for item <- items} + {#if item.__struct__ == AshHq.Docs.Guide} + +
+
+
+ +
+
+
+ + {item.library_name} + + {item_type(item)} +
+
+
+ {item_name(item)} +
+
+
+ {first_sentence(item)} +
+
+
+
+
+
+ + {/if} {#if item.__struct__ != AshHq.Docs.Guide}
-
- {#for {path_item, index} <- Enum.with_index(item_path(item))} - {#if index != 0} - - {/if} -
- {path_item} +
+
+ + {item.library_name} + + {item_type(item)} +
+
+
+ {item_name(item)}
- {/for} - -
- {item_name(item)} +
+
+ {first_sentence(item)}
-
- {raw(item.search_headline)} +
{/if} - {#if item.__struct__ == AshHq.Docs.Guide} - -
-
-
- -
-
- {#for {path_item, index} <- Enum.with_index(item_path(item))} - {#if index != 0} - - {/if} -
- {path_item} -
- {/for} - -
- {item_name(item)} -
-
-
-
- {raw(item.search_headline)} -
-
- - {/if} {/for}
""" end - defp item_name(%{thread_name: thread_name, channel_name: channel_name}), - do: "#{String.capitalize(channel_name)} Forum: #{inspect(thread_name)}" + defp first_sentence(%{text: text}), do: first_sentence(text) + defp first_sentence(%{doc: doc}), do: first_sentence(doc) + defp first_sentence(doc) do + first_sentence = + doc + |> String.trim() + |> String.split("", parts: 2) + |> List.last() + |> String.trim() + |> String.split("\n", parts: 2) + |> Enum.at(0) + |> String.trim() + + if String.starts_with?(first_sentence, "`") do + "" + else + first_sentence + end + end + + defp item_name(%AshHq.Docs.Function{call_name: call_name, arity: arity}), + do: call_name <> "/#{arity}" + + defp item_name(%AshHq.Docs.Option{path: path, name: name}), do: Enum.join(path ++ [name], ".") + defp item_name(%AshHq.Docs.Dsl{path: path, name: name}), do: Enum.join(path ++ [name], ".") defp item_name(%{name: name}), do: name defp item_name(%{version: version}), do: version - defp item_path(%{ - library_name: library_name, - extension_name: extension_name, - path: path - }) do - [library_name, extension_name, path] |> List.flatten() - end - - defp item_path(%{ - library_name: library_name, - module_name: module_name - }) do - [library_name, module_name] - end - - defp item_path(%{library_name: library_name}) do - [library_name] - end - - defp item_path(%{library_version: %{library_name: library_name}}) do - [library_name] - end - - defp item_path(_) do - [] - end - def mount(socket) do {:ok, socket} end @@ -273,19 +271,14 @@ defmodule AshHqWeb.Components.Search do if socket.assigns.search in [nil, ""] do socket else - %{result: item_list} = - AshHq.Docs.Search.run!( - socket.assigns.search, - %{types: socket.assigns[:selected_types]} - ) - - item_list = Enum.take(item_list, 50) - - selected_item = Enum.at(item_list, 0) + item_list = + socket.assigns.search + |> AshHq.Docs.Indexer.search() + |> Enum.take(50) socket |> assign(:item_list, item_list) - |> set_selected_item(selected_item) + |> set_selected_item(Enum.at(item_list, 0)) end end diff --git a/lib/ash_hq_web/pages/ashley.ex b/lib/ash_hq_web/pages/ashley.ex deleted file mode 100644 index 0e80190..0000000 --- a/lib/ash_hq_web/pages/ashley.ex +++ /dev/null @@ -1,353 +0,0 @@ -defmodule AshHqWeb.Pages.Ashley do - @moduledoc "Ashley page" - use Surface.LiveComponent - import AshHqWeb.Tails - - alias Phoenix.LiveView.JS - alias Surface.Components.Form - - alias Surface.Components.Form.{ - Field, - TextArea, - TextInput - } - - prop(current_user, :any) - prop(params, :map) - - data(messages, :list) - data(message_form, :any) - data(new_message_form, :any) - data(conversation, :any) - data(conversations, :list) - data(editing_conversation, :boolean) - data(conversation_form, :any) - - def render(assigns) do - ~F""" -
- {#if is_nil(@current_user) || !@current_user.ashley_access} - You do not have access to this page. - {#else} - -
-
- - New - - -
- -
-
- {#if @conversation} - {#if @editing_conversation} -
- - - - -
- {#else} -
-
- {@conversation.name} -
-
- - - -
-
- {/if} -
-
-
-
- Hello! My name is Ashley. I've been instructed to answer your questions as factually as possible, but I am *far* from perfect. - My code snippets especially are not likely to be accurate. However, I cite my sources below each answer to show you what content - I thought was relevant, so please use that for official clarification. -
-
-
- {#for question <- @conversation.questions} -
-
- {question.question} -
-
-
- {raw(question.answer_html)} -
- {#if question.sources != []} -

Sources

- - {/if} -
- {/for} -
-
-
-
-
-
- {@conversation.question_count} of {AshHq.Ashley.Conversation.conversation_limit()} used - -