diff --git a/VERSION b/VERSION index ac166155..8cf7c24c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.24 +0.2.25 diff --git a/lib/supavisor.ex b/lib/supavisor.ex index 7ad22dd9..ab7e2d04 100644 --- a/lib/supavisor.ex +++ b/lib/supavisor.ex @@ -80,6 +80,35 @@ defmodule Supavisor do end end + @doc """ + During netsplits, or due to certain internal conflicts, :syn may store inconsistent data across the cluster. + This function terminates all connection trees related to a specific tenant. + """ + @spec dirty_terminate(String.t(), pos_integer()) :: map() + def dirty_terminate(tenant, timeout \\ 15_000) do + Registry.lookup(Supavisor.Registry.TenantSups, tenant) + |> Enum.reduce(%{}, fn {pid, %{user: user}}, acc -> + stop = + try do + Supervisor.stop(pid, :shutdown, timeout) + catch + error, reason -> {:error, {error, reason}} + end + + resp = %{ + stop: stop, + cache: del_all_cache(tenant, user) + } + + Map.put(acc, user, resp) + end) + end + + @spec del_all_cache(String.t(), String.t()) :: map() + def del_all_cache(tenant, user) do + %{secrets: Cachex.del(Supavisor.Cache, {:secrets, tenant, user})} + end + @spec get_local_pool(String.t(), String.t()) :: pid() | nil def get_local_pool(tenant, user_alias) do case Registry.lookup(@registry, {:pool, {tenant, user_alias}}) do diff --git a/lib/supavisor/application.ex b/lib/supavisor/application.ex index c8783110..d834821a 100644 --- a/lib/supavisor/application.ex +++ b/lib/supavisor/application.ex @@ -52,22 +52,15 @@ defmodule Supavisor.Application do :syn.set_event_handler(Supavisor.SynHandler) :syn.add_node_to_scopes([:tenants]) - Registry.start_link( - keys: :unique, - name: Supavisor.Registry.Tenants - ) - - Registry.start_link( - keys: :unique, - name: Supavisor.Registry.ManagerTables - ) - PromEx.set_metrics_tags() topologies = Application.get_env(:libcluster, :topologies) || [] children = [ PromEx, + {Registry, keys: :unique, name: Supavisor.Registry.Tenants}, + {Registry, keys: :unique, name: Supavisor.Registry.ManagerTables}, + {Registry, keys: :duplicate, name: Supavisor.Registry.TenantSups}, {Cluster.Supervisor, [topologies, [name: Supavisor.ClusterSupervisor]]}, Supavisor.Repo, # Start the Telemetry supervisor diff --git a/lib/supavisor/tenant_supervisor.ex b/lib/supavisor/tenant_supervisor.ex index a4499234..5fccb640 100644 --- a/lib/supavisor/tenant_supervisor.ex +++ b/lib/supavisor/tenant_supervisor.ex @@ -33,6 +33,7 @@ defmodule Supavisor.TenantSupervisor do {Manager, args} ] + Registry.register(Supavisor.Registry.TenantSups, args.tenant, %{user: args.user_alias}) Supervisor.init(children, strategy: :one_for_all, max_restarts: 10, max_seconds: 60) end diff --git a/lib/supavisor_web/controllers/tenant_controller.ex b/lib/supavisor_web/controllers/tenant_controller.ex index 1c696647..06d7d7d2 100644 --- a/lib/supavisor_web/controllers/tenant_controller.ex +++ b/lib/supavisor_web/controllers/tenant_controller.ex @@ -173,4 +173,23 @@ defmodule SupavisorWeb.TenantController do send_resp(conn, code, "") end + + def terminate(conn, %{"external_id" => external_id}) do + Logger.metadata(project: external_id) + + result = + [node() | Node.list()] + |> Task.async_stream( + fn node -> + %{node => :rpc.call(node, Supavisor, :dirty_terminate, [external_id], 60_000)} + end, + timeout: :infinity + ) + |> Enum.reduce([], fn resp, acc -> + [inspect(resp) | acc] + end) + + Logger.warning("Terminate #{external_id}: #{inspect(result)}") + render(conn, "show_terminate.json", result: result) + end end diff --git a/lib/supavisor_web/router.ex b/lib/supavisor_web/router.ex index 94e40ed8..f0f93ab5 100644 --- a/lib/supavisor_web/router.ex +++ b/lib/supavisor_web/router.ex @@ -46,10 +46,10 @@ defmodule SupavisorWeb.Router do scope "/api", SupavisorWeb do pipe_through(:api) - get("/tenants", TenantController, :index) get("/tenants/:external_id", TenantController, :show) put("/tenants/:external_id", TenantController, :update) delete("/tenants/:external_id", TenantController, :delete) + get("/tenants/:external_id/terminate", TenantController, :terminate) end scope "/metrics", SupavisorWeb do diff --git a/lib/supavisor_web/views/tenant_view.ex b/lib/supavisor_web/views/tenant_view.ex index c5fb9813..f9b86270 100644 --- a/lib/supavisor_web/views/tenant_view.ex +++ b/lib/supavisor_web/views/tenant_view.ex @@ -33,4 +33,8 @@ defmodule SupavisorWeb.TenantView do def render("error.json", %{error: reason}) do %{error: reason} end + + def render("show_terminate.json", %{result: result}) do + %{result: result} + end end diff --git a/test/supavisor_web/controllers/tenant_controller_test.exs b/test/supavisor_web/controllers/tenant_controller_test.exs index fdd4f908..c1b9032c 100644 --- a/test/supavisor_web/controllers/tenant_controller_test.exs +++ b/test/supavisor_web/controllers/tenant_controller_test.exs @@ -103,14 +103,6 @@ defmodule SupavisorWeb.TenantControllerTest do end end - describe "list tenants" do - test "renders tenants", %{conn: conn} do - conn = get(conn, Routes.tenant_path(conn, :index)) - data = json_response(conn, 200)["data"] - assert is_list(data) - end - end - describe "delete tenant" do setup [:create_tenant]