improvement: various docs fixes, cache libraries

config/prod.exs

@@ -17,7 +17,6 @@ config :ash_hq, AshHqWeb.Endpoint,
 config :logger, level: :info
 
 config :ash_hq, :analytics?, true
-config :ash_hq, :periodic_imports, true
 
 config :ash_hq, :download_ua_on_start, true
 

config/runtime.exs

@@ -17,6 +17,8 @@ config :ash_hq, :github,
   client_secret: System.get_env("GITHUB_CLIENT_SECRET"),
   redirect_uri: System.get_env("GITHUB_REDIRECT_URI")
 
+config :ash_hq, :periodic_imports, System.get_env("PERIODIC_IMPORTS") == "true"
+
 host = System.get_env("PHX_HOST") || "localhost"
 port = String.to_integer(System.get_env("PORT") || "4000")
 

lib/ash_hq/accounts/resources/user/user.ex

@@ -11,118 +11,117 @@ defmodule AshHq.Accounts.User do
   require Ash.Query
 
   authentication do
-    api(AshHq.Accounts)
+    api AshHq.Accounts
 
     strategies do
       password :password do
-        identity_field(:email)
-        hashed_password_field(:hashed_password)
+        identity_field :email
+        hashed_password_field :hashed_password
 
         resettable do
-          sender(AshHq.Accounts.User.Senders.SendPasswordResetEmail)
+          sender AshHq.Accounts.User.Senders.SendPasswordResetEmail
         end
       end
 
       github do
-        client_id(AshHq.Accounts.Secrets)
-        client_secret(AshHq.Accounts.Secrets)
-        redirect_uri(AshHq.Accounts.Secrets)
+        client_id AshHq.Accounts.Secrets
+        client_secret AshHq.Accounts.Secrets
+        redirect_uri AshHq.Accounts.Secrets
       end
     end
 
     tokens do
-      enabled?(true)
-      token_resource(AshHq.Accounts.UserToken)
-      signing_secret(AshHq.Accounts.Secrets)
-      store_all_tokens?(true)
-      require_token_presence_for_authentication?(true)
+      enabled? true
+      token_resource AshHq.Accounts.UserToken
+      signing_secret AshHq.Accounts.Secrets
+      store_all_tokens? true
+      require_token_presence_for_authentication? true
     end
 
     add_ons do
       confirmation :confirm do
-        monitor_fields([:email])
+        monitor_fields [:email]
 
-        sender(AshHq.Accounts.User.Senders.SendConfirmationEmail)
+        sender AshHq.Accounts.User.Senders.SendConfirmationEmail
       end
     end
   end
 
   attributes do
-    uuid_primary_key(:id)
+    uuid_primary_key :id
 
-    attribute(:email, :ci_string,
+    attribute :email, :ci_string,
       allow_nil?: false,
       constraints: [
         max_length: 160
       ]
-    )
 
-    attribute(:hashed_password, :string, private?: true, sensitive?: true)
+    attribute :hashed_password, :string, private?: true, sensitive?: true
 
-    attribute(:encrypted_name, AshHq.Types.EncryptedString)
-    attribute(:encrypted_address, AshHq.Types.EncryptedString)
-    attribute(:shirt_size, :string)
-    attribute(:github_info, :map)
+    attribute :encrypted_name, AshHq.Types.EncryptedString
+    attribute :encrypted_address, AshHq.Types.EncryptedString
+    attribute :shirt_size, :string
+    attribute :github_info, :map
 
-    create_timestamp(:created_at)
-    update_timestamp(:updated_at)
+    create_timestamp :created_at
+    update_timestamp :updated_at
   end
 
   relationships do
     has_one :token, AshHq.Accounts.UserToken do
-      destination_attribute(:user_id)
-      private?(true)
+      destination_attribute :user_id
+      private? true
     end
   end
 
   policies do
     bypass AshAuthentication.Checks.AshAuthenticationInteraction do
-      authorize_if(always())
+      authorize_if always()
     end
 
     policy action(:read) do
-      authorize_if(expr(id == ^actor(:id)))
+      authorize_if expr(id == ^actor(:id))
     end
 
     policy action(:update_email) do
-      description("A logged in user can update their email")
-      authorize_if(expr(id == ^actor(:id)))
+      description "A logged in user can update their email"
+      authorize_if expr(id == ^actor(:id))
     end
 
     policy action(:resend_confirmation_instructions) do
-      description("A logged in user can request an email confirmation")
-      authorize_if(expr(id == ^actor(:id)))
+      description "A logged in user can request an email confirmation"
+      authorize_if expr(id == ^actor(:id))
     end
 
     policy action(:change_password) do
-      description("A logged in user can reset their password")
-      authorize_if(expr(id == ^actor(:id)))
+      description "A logged in user can reset their password"
+      authorize_if expr(id == ^actor(:id))
     end
 
     policy action(:update_merch_settings) do
-      description("A logged in user can update their merch settings")
-      authorize_if(expr(id == ^actor(:id)))
+      description "A logged in user can update their merch settings"
+      authorize_if expr(id == ^actor(:id))
     end
   end
 
   postgres do
-    table("users")
-    repo(AshHq.Repo)
+    table "users"
+    repo AshHq.Repo
   end
 
   actions do
-    defaults([:read])
+    defaults [:read]
 
     create :register_with_github do
       argument :user_info, :map do
-        allow_nil?(false)
+        allow_nil? false
       end
 
       argument :oauth_tokens, :map do
-        allow_nil?(false)
+        allow_nil? false
       end
 
-      change(fn changeset, _ ->
+      change fn changeset, _ ->
         user_info = Ash.Changeset.get_argument(changeset, :user_info)
 
         changeset =
@@ -139,62 +138,62 @@ defmodule AshHq.Accounts.User do
         changeset
         |> Ash.Changeset.change_attribute(:email, Map.get(user_info, "email"))
         |> Ash.Changeset.change_attribute(:github_info, user_info)
-      end)
+      end
 
-      change(AshAuthentication.GenerateTokenChange)
-      upsert?(true)
-      upsert_identity(:unique_email)
+      change AshAuthentication.GenerateTokenChange
+      upsert? true
+      upsert_identity :unique_email
     end
 
     update :change_password do
-      accept([])
+      accept []
 
       argument :current_password, :string do
-        sensitive?(true)
-        allow_nil?(false)
+        sensitive? true
+        allow_nil? false
       end
 
       argument :password, :string do
-        sensitive?(true)
-        allow_nil?(false)
+        sensitive? true
+        allow_nil? false
       end
 
       argument :password_confirmation, :string do
-        sensitive?(true)
-        allow_nil?(false)
+        sensitive? true
+        allow_nil? false
       end
 
-      validate(confirm(:password, :password_confirmation))
+      validate confirm(:password, :password_confirmation)
 
       validate {AshAuthentication.Strategy.Password.PasswordValidation,
                 password_argument: :current_password} do
-        only_when_valid?(true)
-        before_action?(true)
+        only_when_valid? true
+        before_action? true
       end
 
-      change(set_context(%{strategy_name: :password}))
-      change(AshAuthentication.Strategy.Password.HashPasswordChange)
+      change set_context(%{strategy_name: :password})
+      change AshAuthentication.Strategy.Password.HashPasswordChange
     end
 
     update :update_email do
-      accept([:email])
+      accept [:email]
 
       argument :current_password, :string do
-        sensitive?(true)
-        allow_nil?(false)
+        sensitive? true
+        allow_nil? false
       end
 
       validate {AshAuthentication.Strategy.Password.PasswordValidation,
                 password_argument: :current_password} do
-        only_when_valid?(true)
-        before_action?(true)
+        only_when_valid? true
+        before_action? true
       end
     end
 
     update :resend_confirmation_instructions do
-      accept([])
+      accept []
 
-      change(fn changeset, _context ->
+      change fn changeset, _context ->
         Ash.Changeset.before_action(changeset, fn changeset ->
           case AshHq.Accounts.UserToken.email_token_for_user(changeset.data.id,
                  authorize?: false
@@ -222,49 +221,48 @@ defmodule AshHq.Accounts.User do
               Ash.Changeset.add_error(changeset, "Could not determine what email to use")
           end
         end)
-      end)
+      end
     end
 
     update :update_merch_settings do
-      argument(:address, :string)
-      argument(:name, :string)
+      argument :address, :string
+      argument :name, :string
 
-      accept([:shirt_size])
-      change(set_attribute(:encrypted_address, arg(:address)))
-      change(set_attribute(:encrypted_name, arg(:name)))
+      accept [:shirt_size]
+      change set_attribute(:encrypted_address, arg(:address))
+      change set_attribute(:encrypted_name, arg(:name))
     end
   end
 
   code_interface do
-    define_for(AshHq.Accounts)
-    define(:resend_confirmation_instructions)
-    define(:register_with_password, args: [:email, :password, :password_confirmation])
+    define_for AshHq.Accounts
+    define :resend_confirmation_instructions
+    define :register_with_password, args: [:email, :password, :password_confirmation]
   end
 
   resource do
-    description("""
+    description """
     Represents the user of a system.
-    """)
+    """
   end
 
   identities do
     identity :unique_email, [:email] do
-      eager_check_with(AshHq.Accounts)
+      eager_check_with AshHq.Accounts
     end
   end
 
   changes do
-    change(AshHq.Accounts.User.Changes.RemoveAllTokens,
+    change AshHq.Accounts.User.Changes.RemoveAllTokens,
       where: [action_is(:password_reset_with_password)]
-    )
   end
 
   validations do
-    validate(match(:email, ~r/^[^\s]+@[^\s]+$/), message: "must have the @ sign and no spaces")
+    validate match(:email, ~r/^[^\s]+@[^\s]+$/), message: "must have the @ sign and no spaces"
   end
 
   calculations do
-    calculate(:address, :string, decrypt(:encrypted_address))
-    calculate(:name, :string, decrypt(:encrypted_name))
+    calculate :address, :string, decrypt(:encrypted_address)
+    calculate :name, :string, decrypt(:encrypted_name)
   end
 end

lib/ash_hq/application.ex

@@ -34,7 +34,9 @@ defmodule AshHq.Application do
         {Phoenix.PubSub, name: AshHq.PubSub},
         {AshHq.Docs.Cache, []},
         # Start the Endpoint (http/https)
-        AshHqWeb.Endpoint
+        AshHqWeb.Endpoint,
+        {AshHq.Docs.Library.Agent, nil}
+
         # Start a worker by calling: AshHq.Worker.start_link(arg)
         # {AshHq.Worker, arg}
       ] ++ importer

lib/ash_hq/docs/extensions/render_markdown/highlighter.ex

@@ -19,6 +19,33 @@ defmodule AshHq.Docs.Extensions.RenderMarkdown.Highlighter do
       {"a", attrs, contents} ->
         {"a", rewrite_href_attr(attrs, current_library, libraries), contents}
 
+      {"pre", _, [{:keep, contents}]} ->
+        {:keep, ~s(<pre class="code-pre">#{contents}</pre>)}
+
+      {"pre", _, [{"code", attrs, [body]}]} when is_binary(body) ->
+        lexer =
+          find_value_class(attrs, fn class ->
+            case Makeup.Registry.fetch_lexer_by_name(class) do
+              {:ok, {lexer, opts}} -> {class, lexer, opts}
+              :error -> nil
+            end
+          end)
+
+        code =
+          case lexer do
+            {lang, lexer, opts} ->
+              render_code(lang, lexer, opts, body)
+
+            nil ->
+              if find_value_class(attrs, &(&1 == "inline")) do
+                maybe_highlight_module(body, libraries, current_module)
+              else
+                ~s(<code class="text-black dark:text-white">#{body}</code>)
+              end
+          end
+
+        {:keep, ~s(<pre class="code-pre">#{code}</pre>)}
+
       {"code", attrs, [body]} when is_binary(body) ->
         lexer =
           find_value_class(attrs, fn class ->
@@ -60,7 +87,7 @@ defmodule AshHq.Docs.Extensions.RenderMarkdown.Highlighter do
   def rewrite_href(value, current_library, libraries) do
     uri = URI.parse(value)
 
-    case {uri, Path.split(String.trim_leading(uri.path || "", "/"))} |> IO.inspect() do
+    case {uri, Path.split(String.trim_leading(uri.path || "", "/"))} do
       {%{host: "hexdocs.pm"}, [library, guide]} ->
         if Enum.any?(libraries, &(&1.name == library)) do
           if String.ends_with?(guide, ".html") do
@@ -385,7 +412,13 @@ defmodule AshHq.Docs.Extensions.RenderMarkdown.Highlighter do
       -String.length(match)
     end)
     |> Enum.at(0)
-    |> elem(0)
+    |> case do
+      {library, _match} ->
+        library
+
+      _ ->
+        nil
+    end
   end
 
   defp render_code(lang, lexer, lexer_opts, code) do

lib/ash_hq/docs/resources/dsl/dsl.ex

@@ -29,9 +29,11 @@ defmodule AshHq.Docs.Dsl do
 
     attribute :examples, {:array, :string}
     attribute :args, {:array, :string}
+
     attribute :optional_args, {:array, :string} do
       default []
     end
+
     attribute :arg_defaults, :map
     attribute :path, {:array, :string}
     attribute :recursive_as, :string

lib/ash_hq/docs/resources/library/agent.ex

@@ -0,0 +1,26 @@
+defmodule AshHq.Docs.Library.Agent do
+  use Agent
+
+  def start_link(_) do
+    Agent.start_link(fn -> nil end, name: __MODULE__)
+  end
+
+  def get() do
+    Agent.get_and_update(__MODULE__, fn state ->
+      state =
+        if state == nil do
+          AshHq.Docs.Library.read!(%{check_cache: false})
+        else
+          state
+        end
+
+      {state, state}
+    end)
+  end
+
+  def clear() do
+    Agent.update(__MODULE__, fn _state ->
+      nil
+    end)
+  end
+end

lib/ash_hq/docs/resources/library/library.ex

@@ -47,7 +47,36 @@ defmodule AshHq.Docs.Library do
   end
 
   actions do
-    defaults [:read, :create, :update, :destroy]
+    defaults [:create, :update, :destroy]
+
+    read :read do
+      primary? true
+
+      argument :check_cache, :boolean do
+        default true
+      end
+
+      prepare fn query, _ ->
+        if Ash.Query.get_argument(query, :check_cache) do
+          Ash.Query.before_action(query, fn query ->
+            AshHq.Docs
+            |> Ash.Filter.Runtime.filter_matches(
+              AshHq.Docs.Library.Agent.get(),
+              query.filter
+            )
+            |> case do
+              {:ok, results} ->
+                Ash.Query.set_result(query, {:ok, Ash.Sort.runtime_sort(results, query.sort)})
+
+              {:error, _} ->
+                query
+            end
+          end)
+        else
+          query
+        end
+      end
+    end
 
     read :by_name do
       argument :name, :string do
@@ -75,6 +104,15 @@ defmodule AshHq.Docs.Library do
     identity :unique_name, [:name]
   end
 
+  changes do
+    change fn changeset, _ ->
+      Ash.Changeset.after_action(changeset, fn _changeset, result ->
+        AshHq.Docs.Library.Agent.clear()
+        {:ok, result}
+      end)
+    end
+  end
+
   aggregates do
     first :latest_version, :versions, :version do
       sort version: :desc

mix.lock

@@ -1,7 +1,7 @@
 %{
   "absinthe": {:hex, :absinthe, "1.7.0", "36819e7b1fd5046c9c734f27fe7e564aed3bda59f0354c37cd2df88fd32dd014", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0 or ~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "566a5b5519afc9b29c4d367f0c6768162de3ec03e9bf9916f9dc2bcbe7c09643"},
   "absinthe_plug": {:hex, :absinthe_plug, "1.5.8", "38d230641ba9dca8f72f1fed2dfc8abd53b3907d1996363da32434ab6ee5d6ab", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bbb04176647b735828861e7b2705465e53e2cf54ccf5a73ddd1ebd855f996e5a"},
-  "ash": {:git, "https://github.com/ash-project/ash.git", "22e07c06b5e0da531e7e6f632009290031429875", []},
+  "ash": {:git, "https://github.com/ash-project/ash.git", "e3a4f51d67232223584fe0ecf5154dd9885247a3", []},
   "ash_admin": {:git, "https://github.com/ash-project/ash_admin.git", "cdb3b469abeeadddda884e7cfcf67c6fea10f9ef", []},
   "ash_authentication": {:git, "https://github.com/team-alembic/ash_authentication.git", "00f082cc8dfa0f56d3e2dec3eaf178a58d6d4b4a", []},
   "ash_authentication_phoenix": {:git, "https://github.com/team-alembic/ash_authentication_phoenix.git", "14c38739cf56c4821797963b9ee261597a261b00", []},

priv/repo/migrations/20230115062845_migrate_resources40.exs

@@ -9,15 +9,15 @@ defmodule AshHq.Repo.Migrations.MigrateResources40 do
 
   def up do
     alter table(:dsls) do
-      add :optional_args, {:array, :text}, default: []
-      add :arg_defaults, :map
+      add(:optional_args, {:array, :text}, default: [])
+      add(:arg_defaults, :map)
     end
   end
 
   def down do
     alter table(:dsls) do
-      remove :arg_defaults
-      remove :optional_args
+      remove(:arg_defaults)
+      remove(:optional_args)
     end
   end
-end
+end

priv/repo/migrations/20230117145347_migrate_resources41.exs

@@ -9,21 +9,21 @@ defmodule AshHq.Repo.Migrations.MigrateResources41 do
 
   def up do
     alter table(:library_versions) do
-      add :hydrated, :boolean, null: false, default: false
+      add(:hydrated, :boolean, null: false, default: false)
     end
 
     alter table(:libraries) do
-      add :module_prefixes, {:array, :text}, null: false, default: []
+      add(:module_prefixes, {:array, :text}, null: false, default: [])
     end
   end
 
   def down do
     alter table(:libraries) do
-      remove :module_prefixes
+      remove(:module_prefixes)
     end
 
     alter table(:library_versions) do
-      remove :hydrated
+      remove(:hydrated)
     end
   end
-end
+end

priv/repo/migrations/20230117205917_migrate_resources42.exs

@@ -9,7 +9,7 @@ defmodule AshHq.Repo.Migrations.MigrateResources42 do
 
   def up do
     alter table(:libraries) do
-      add :mix_project, :text
+      add(:mix_project, :text)
     end
 
     # alter table(:dsls) do
@@ -32,7 +32,7 @@ defmodule AshHq.Repo.Migrations.MigrateResources42 do
     # end
     # 
     alter table(:libraries) do
-      remove :mix_project
+      remove(:mix_project)
     end
   end
-end
+end

priv/scripts/build_dsl_docs.exs

@@ -365,23 +365,24 @@ defmodule Utils do
         callbacks = Types.callbacks_for_module(module)
         typespecs = Types.specs_for_module(module)
 
-        {:ok, %{
-          name: inspect(module),
-          doc: module_doc,
-          file: file,
-          order: order,
-          category: category,
-          functions:
-            defs
-            |> Enum.with_index()
-            |> Enum.flat_map(fn {definition, i} ->
-              build_function(definition, file, types, callbacks, typespecs, i)
-            end)
-        }}
+        {:ok,
+         %{
+           name: inspect(module),
+           doc: module_doc,
+           file: file,
+           order: order,
+           category: category,
+           functions:
+             defs
+             |> Enum.with_index()
+             |> Enum.flat_map(fn {definition, i} ->
+               build_function(definition, file, types, callbacks, typespecs, i)
+             end)
+         }}
+
       _ ->
         :error
     end
-
   end
 
   def build_mix_task(mix_task, category, order) do
@@ -500,6 +501,7 @@ defmodule Utils do
 
   def guides(mix_project, name) do
     root_dir = File.cwd!()
+
     app_dir =
       name
       |> Application.app_dir()
@@ -510,17 +512,36 @@ defmodule Utils do
       File.cd!(app_dir)
 
       extras =
-        Enum.map(mix_project.project[:docs][:extras], fn {file, config} ->
-          config =
-          if config[:title] do
-            Keyword.put(config, :name, config[:title])
-          else
-            config
-          end
+        Enum.map(mix_project.project[:docs][:extras], fn
+          {file, config} ->
+            file = to_string(file)
 
-          {to_string(file), config}
+            config =
+              if config[:title] do
+                Keyword.put(config, :name, config[:title])
+              else
+                title =
+                  file
+                  |> Path.basename(".md")
+                  |> String.split(~r/[-_]/)
+                  |> Enum.map(&String.capitalize/1)
+                  |> Enum.join(" ")
+                  |> case do
+                    "F A Q" ->
+                      "FAQ"
+
+                    other ->
+                      other
+                  end
+
+                Keyword.put(config, :name, title)
+              end
+
+            {to_string(file), config}
 
           file ->
+            file = to_string(file)
+
             title =
               file
               |> Path.basename(".md")
@@ -535,7 +556,7 @@ defmodule Utils do
                   other
               end
 
-            {to_string(file), name: title}
+            {file, name: title}
         end)
 
       groups_for_extras =
@@ -597,7 +618,8 @@ acc = %{
   mix_tasks: []
 }
 
-extensions = mix_project.project[:docs][:spark][:extensions] || mix_project.project[:docs][:spark_extensions]
+extensions =
+  mix_project.project[:docs][:spark][:extensions] || mix_project.project[:docs][:spark_extensions]
 
 {:ok, all_modules} =
   name
@@ -624,14 +646,15 @@ all_modules =
 
 acc =
   mix_project.project[:docs][:groups_for_modules]
-  |> Enum.reject(fn {"Internals", _} ->
-    true
+  |> Enum.reject(fn
+    {"Internals", _} ->
+      true
+
     _ ->
       false
   end)
   |> Enum.reduce(acc, fn {category, modules}, acc ->
-    modules =
-      Utils.modules_for(all_modules, modules)
+    modules = Utils.modules_for(all_modules, modules)
 
     modules
     |> Enum.with_index()
@@ -641,6 +664,7 @@ acc =
           Map.update!(acc, :modules, fn modules ->
             [built | modules]
           end)
+
         _ ->
           acc
       end