feat: api endpoint to terminate tenancy tree connection (#145)

VERSION

@@ -1 +1 @@
-0.2.24
+0.2.25

lib/supavisor.ex

@@ -80,6 +80,35 @@ defmodule Supavisor do
     end
   end
 
+  @doc """
+  During netsplits, or due to certain internal conflicts, :syn may store inconsistent data across the cluster.
+  This function terminates all connection trees related to a specific tenant.
+  """
+  @spec dirty_terminate(String.t(), pos_integer()) :: map()
+  def dirty_terminate(tenant, timeout \\ 15_000) do
+    Registry.lookup(Supavisor.Registry.TenantSups, tenant)
+    |> Enum.reduce(%{}, fn {pid, %{user: user}}, acc ->
+      stop =
+        try do
+          Supervisor.stop(pid, :shutdown, timeout)
+        catch
+          error, reason -> {:error, {error, reason}}
+        end
+
+      resp = %{
+        stop: stop,
+        cache: del_all_cache(tenant, user)
+      }
+
+      Map.put(acc, user, resp)
+    end)
+  end
+
+  @spec del_all_cache(String.t(), String.t()) :: map()
+  def del_all_cache(tenant, user) do
+    %{secrets: Cachex.del(Supavisor.Cache, {:secrets, tenant, user})}
+  end
+
   @spec get_local_pool(String.t(), String.t()) :: pid() | nil
   def get_local_pool(tenant, user_alias) do
     case Registry.lookup(@registry, {:pool, {tenant, user_alias}}) do

lib/supavisor/application.ex

@@ -52,22 +52,15 @@ defmodule Supavisor.Application do
     :syn.set_event_handler(Supavisor.SynHandler)
     :syn.add_node_to_scopes([:tenants])
 
-    Registry.start_link(
-      keys: :unique,
-      name: Supavisor.Registry.Tenants
-    )
-
-    Registry.start_link(
-      keys: :unique,
-      name: Supavisor.Registry.ManagerTables
-    )
-
     PromEx.set_metrics_tags()
 
     topologies = Application.get_env(:libcluster, :topologies) || []
 
     children = [
       PromEx,
+      {Registry, keys: :unique, name: Supavisor.Registry.Tenants},
+      {Registry, keys: :unique, name: Supavisor.Registry.ManagerTables},
+      {Registry, keys: :duplicate, name: Supavisor.Registry.TenantSups},
       {Cluster.Supervisor, [topologies, [name: Supavisor.ClusterSupervisor]]},
       Supavisor.Repo,
       # Start the Telemetry supervisor

lib/supavisor/tenant_supervisor.ex

@@ -33,6 +33,7 @@ defmodule Supavisor.TenantSupervisor do
       {Manager, args}
     ]
 
+    Registry.register(Supavisor.Registry.TenantSups, args.tenant, %{user: args.user_alias})
     Supervisor.init(children, strategy: :one_for_all, max_restarts: 10, max_seconds: 60)
   end
 

lib/supavisor_web/controllers/tenant_controller.ex

@@ -173,4 +173,23 @@ defmodule SupavisorWeb.TenantController do
 
     send_resp(conn, code, "")
   end
+
+  def terminate(conn, %{"external_id" => external_id}) do
+    Logger.metadata(project: external_id)
+
+    result =
+      [node() | Node.list()]
+      |> Task.async_stream(
+        fn node ->
+          %{node => :rpc.call(node, Supavisor, :dirty_terminate, [external_id], 60_000)}
+        end,
+        timeout: :infinity
+      )
+      |> Enum.reduce([], fn resp, acc ->
+        [inspect(resp) | acc]
+      end)
+
+    Logger.warning("Terminate #{external_id}: #{inspect(result)}")
+    render(conn, "show_terminate.json", result: result)
+  end
 end

lib/supavisor_web/router.ex

@@ -46,10 +46,10 @@ defmodule SupavisorWeb.Router do
   scope "/api", SupavisorWeb do
     pipe_through(:api)
 
-    get("/tenants", TenantController, :index)
     get("/tenants/:external_id", TenantController, :show)
     put("/tenants/:external_id", TenantController, :update)
     delete("/tenants/:external_id", TenantController, :delete)
+    get("/tenants/:external_id/terminate", TenantController, :terminate)
   end
 
   scope "/metrics", SupavisorWeb do

lib/supavisor_web/views/tenant_view.ex

@@ -33,4 +33,8 @@ defmodule SupavisorWeb.TenantView do
   def render("error.json", %{error: reason}) do
     %{error: reason}
   end
+
+  def render("show_terminate.json", %{result: result}) do
+    %{result: result}
+  end
 end

test/supavisor_web/controllers/tenant_controller_test.exs

@@ -103,14 +103,6 @@ defmodule SupavisorWeb.TenantControllerTest do
     end
   end
 
-  describe "list tenants" do
-    test "renders tenants", %{conn: conn} do
-      conn = get(conn, Routes.tenant_path(conn, :index))
-      data = json_response(conn, 200)["data"]
-      assert is_list(data)
-    end
-  end
-
   describe "delete tenant" do
     setup [:create_tenant]