From 2fd65876b32be08930f3a1fda712bca7632cd328 Mon Sep 17 00:00:00 2001 From: Ilya Verbitskiy Date: Tue, 24 Sep 2024 22:41:40 +0200 Subject: [PATCH] bbf: big fucking refactoring This is a big boi that introduces a new architecture with a cluster controller and multiple workers that pull the state from the controller and try to realize the desired state. --- apps/virtuerl/src/virtuerl_app.erl | 6 + apps/virtuerl/src/virtuerl_host.erl | 164 +++++++++++++++ apps/virtuerl/src/virtuerl_img.erl | 25 ++- apps/virtuerl/src/virtuerl_mgt.erl | 208 ++++++++----------- apps/virtuerl/src/virtuerl_mgt_sup.erl | 27 +++ apps/virtuerl/src/virtuerl_net.erl | 123 ++++++----- apps/virtuerl/src/virtuerl_server.erl | 16 +- apps/virtuerl/src/virtuerl_sup.erl | 48 ++--- apps/virtuerl/src/virtuerl_sup_sup.erl | 4 +- apps/virtuerl/test/virtuerl_SUITE.erl | 2 + apps/virtuerl/test/virtuerl_mock_SUITE.erl | 24 ++- apps/virtuerl/test/virtuerl_mock_net.erl | 72 +++++++ apps/virtuerl/test/virtuerl_nested_SUITE.erl | 2 + config/sys.config | 39 ++-- config/test.sys.config | 20 +- 15 files changed, 546 insertions(+), 234 deletions(-) create mode 100644 apps/virtuerl/src/virtuerl_host.erl create mode 100644 apps/virtuerl/src/virtuerl_mgt_sup.erl create mode 100644 apps/virtuerl/test/virtuerl_mock_net.erl diff --git a/apps/virtuerl/src/virtuerl_app.erl b/apps/virtuerl/src/virtuerl_app.erl index b48fbf2..27a6ab7 100644 --- a/apps/virtuerl/src/virtuerl_app.erl +++ b/apps/virtuerl/src/virtuerl_app.erl @@ -27,6 +27,12 @@ start(_StartType, _StartArgs) -> end, [ virtuerl_server:start(Name, Conf) || {Name, Conf} <- maps:to_list(Servers) ], + Clusters = case application:get_env(clusters) of + undefined -> #{}; + {ok, Clusters0} -> Clusters0 + end, + [ virtuerl_mgt_sup:start(Cluster) || Cluster <- Clusters ], + {ok, Pid}. diff --git a/apps/virtuerl/src/virtuerl_host.erl b/apps/virtuerl/src/virtuerl_host.erl new file mode 100644 index 0000000..7845aee --- /dev/null +++ b/apps/virtuerl/src/virtuerl_host.erl @@ -0,0 +1,164 @@ +-module(virtuerl_host). + +-behaviour(gen_server). + +-export([start_link/2, + home_path/0, + sync/1]). +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + handle_continue/2]). + +-include_lib("kernel/include/logger.hrl"). + +-define(APPLICATION, virtuerl). + +-record(state, {server_id, vm_proc_mod, table, idmap, controller, dom_tap_map, prefix}). + +%%%=================================================================== +%%% Spawning and gen_server implementation +%%%=================================================================== + + +home_path() -> + application:get_env(?APPLICATION, home, "var"). + + +sync(Pid) -> + gen_server:cast(Pid, sync). + + +start_link(ServerId, Conf) -> + gen_server:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [ServerId, Conf], []). + + +init([ServerId, Conf]) -> + #{cluster := Cluster, vm_proc_mod := VmProcMod, prefix := IfPrefix} = Conf, + pg:join(Cluster, [self()]), + % TODO: query supervisor and populate dom_tap_map + {ok, #state{server_id = ServerId, vm_proc_mod = VmProcMod, idmap = #{}, dom_tap_map = #{}, prefix = IfPrefix}}. + + +generate_unique_tap_name(Prefix, TapNames) -> + TapName = iolist_to_binary([Prefix, "tap", binary:encode_hex(<<(rand:uniform(16#ffffff)):24>>)]), + case lists:member(TapName, TapNames) of + false -> + TapName; + true -> + generate_unique_tap_name(Prefix, TapNames) + end. + + +handle_continue(sync_domains, #state{server_id = ServerId, vm_proc_mod = VmProcMod, idmap = IdMap, dom_tap_map = DomTapMap, prefix = IfPrefix, controller = ControllerPid} = State) -> + Domains0 = virtuerl_mgt:domains_list(ControllerPid, ServerId), + io:format("DOMAINS0: ~p~n", [Domains0]), + {_, Domains1} = lists:foldl(fun(Dom, {IfNames, Doms}) -> + #{id := DomId} = Dom, + case DomTapMap of + #{DomId := TapName} -> {[TapName | IfNames], [maps:put(tap_name, TapName, Dom) | Doms]}; + _ -> + TapName = generate_unique_tap_name(IfPrefix, IfNames), + {[TapName | IfNames], [maps:put(tap_name, TapName, Dom) | Doms]} + end + end, + {[], []}, + Domains0), + + io:format("DOMAINS1: ~p~n", [Domains1]), + Domains = [ {Id, Dom} || #{id := Id} = Dom <- Domains1 ], + TargetDomains = maps:from_list( + [ {Id, case maps:get(host, Domain, localhost) of localhost -> node(); Else -> Else end} + || {Id, Domain} <- Domains, + case Domain of + #{state := stopped} -> false; + _ -> true + end ]), + + AllNodes = nodes([this]), + RunningDomains = maps:from_list(lists:flatten( + [ [ {Id, Node} || {Id, _, _, _} <- virtuerl_sup:which_children(ServerId, Node), is_binary(Id) ] + || Node <- AllNodes ])), + ToDelete = maps:without(maps:keys(TargetDomains), RunningDomains), + ToAdd = maps:without(maps:keys(RunningDomains), TargetDomains), + [ virtuerl_sup:terminate_child(ServerId, Node, Id) || {Id, Node} <- maps:to_list(ToDelete) ], + [ virtuerl_sup:delete_child(ServerId, Node, Id) || {Id, Node} <- maps:to_list(ToDelete) ], + + % cleanup deleted domains + case file:list_dir(filename:join([virtuerl_host:home_path(), "domains"])) of + {ok, Filenames} -> + DirsToDel = sets:subtract(sets:from_list([ iolist_to_binary(FName) || FName <- Filenames ]), + sets:from_list(maps:keys(maps:from_list(Domains)))), + [ file:del_dir_r(filename:join([virtuerl_host:home_path(), "domains", Dir])) || Dir <- sets:to_list(DirsToDel) ]; + _ -> ok + end, + + % lists:foreach(fun () -> ok end, List) + DomsByNode0 = maps:groups_from_list( + fun({_Id, Dom}) -> case maps:get(host, Dom, localhost) of localhost -> node(); Else -> Else end end, + fun({_Id, Dom}) -> Dom end, + Domains), + DomsByNode = maps:merge(#{node() => []}, DomsByNode0), + [ virtuerl_net:update_net(Node, ServerId, Doms) || {Node, Doms} <- maps:to_list(DomsByNode), lists:member(Node, AllNodes) ], + + io:format("DomsByNode: ~p~n", [DomsByNode]), + io:format("RUNNING: ~p~n", [RunningDomains]), + io:format("TOADD: ~p~n", [ToAdd]), + io:format("TODELETE: ~p~n", [ToDelete]), + [ virtuerl_mgt:notify(ControllerPid, {domain_stopped, DomId}) || DomId <- maps:keys(ToDelete) ], + + VmPids = [ {Id, + virtuerl_sup:start_child(ServerId, + Node, + {Id, + {VmProcMod, start_link, [maps:get(Id, maps:from_list(Domains))]}, + transient, + infinity, + worker, + []})} || {Id, Node} <- maps:to_list(ToAdd), lists:member(Node, AllNodes) ], + io:format("VMPIDS: ~p~n", [VmPids]), + [ virtuerl_mgt:notify(ControllerPid, {domain_started, DomId}) || {DomId, _} <- VmPids ], + VmPidToDomId = maps:from_list([ {VmPid, DomId} || {DomId, {ok, VmPid}} <- VmPids ]), + [ monitor(process, VmPid) || {_, {ok, VmPid}} <- VmPids ], + + NewDomTapMap = maps:from_list([ {Id, TapName} || {Id, #{tap_name := TapName}} <- Domains ]), + {noreply, State#state{idmap = maps:merge(IdMap, VmPidToDomId), dom_tap_map = NewDomTapMap}}. + + +handle_call(_Request, _From, _State) -> + erlang:error(not_implemented). + + +handle_cast(sync, State) -> + {noreply, State, {continue, sync_domains}}; +handle_cast(_Request, _State) -> + erlang:error(not_implemented). + + +handle_info({enslave, ControllerPid}, State) -> + ?LOG_NOTICE(#{who => ?MODULE, msg => "got enslaved!", controller => ControllerPid}), + {noreply, State#state{controller = ControllerPid}, {continue, sync_domains}}; +handle_info({'DOWN', _, process, Pid, normal}, #state{idmap = IdMap} = State) -> + NewIdMap = case IdMap of + #{Pid := DomId} -> + virtuerl_mgt:domain_stop(DomId), + maps:remove(Pid, IdMap); + #{} -> + ?LOG_WARNING(#{module => ?MODULE, msg => "process down but not in registry", pid => Pid}), + IdMap + end, + {noreply, State#state{idmap = NewIdMap}}; +handle_info(Info, State) -> + ?LOG_NOTICE(#{module => ?MODULE, msg => "unhandled info message", info => Info}), + {noreply, State}. + + +terminate(_Reason, _State) -> + ok. + + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/virtuerl/src/virtuerl_img.erl b/apps/virtuerl/src/virtuerl_img.erl index 32df6f5..6eb3997 100644 --- a/apps/virtuerl/src/virtuerl_img.erl +++ b/apps/virtuerl/src/virtuerl_img.erl @@ -2,7 +2,9 @@ -behaviour(gen_server). --export([start_link/0, list_images/0, ensure_image/1]). +-export([start_link/1, + list_images/0, list_images/1, + ensure_image/1, ensure_image/2]). -export([init/1, handle_call/3, handle_cast/2, @@ -11,24 +13,27 @@ code_change/3]). -export([]). --define(SERVER, ?MODULE). --define(APPLICATION, virtuerl). - %%%=================================================================== %%% Spawning and gen_server implementation %%%=================================================================== -list_images() -> - gen_server:call(?SERVER, list_images). +list_images() -> list_images(default). + + +list_images(ServerId) -> + gen_server:call({via, virtuerl_reg, {ServerId, ?MODULE}}, list_images). + + +ensure_image(ImageName) -> ensure_image(default, ImageName). -ensure_image(ImageName) -> - gen_server:call(?SERVER, {ensure_image, ImageName}, infinity). +ensure_image(ServerId, ImageName) -> + gen_server:call({via, virtuerl_reg, {ServerId, ?MODULE}}, {ensure_image, ImageName}, infinity). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +start_link(ServerId) -> + gen_server:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [], []). init([]) -> diff --git a/apps/virtuerl/src/virtuerl_mgt.erl b/apps/virtuerl/src/virtuerl_mgt.erl index 7e87f26..d00eeb4 100644 --- a/apps/virtuerl/src/virtuerl_mgt.erl +++ b/apps/virtuerl/src/virtuerl_mgt.erl @@ -2,8 +2,10 @@ -behaviour(gen_server). --export([start_link/2, +-export([start/1, + start_link/1, home_path/0, + notify/2, image_from_domain/2, domain_update/1, create_vm/0, @@ -12,7 +14,7 @@ domain_delete/1, domain_stop/1, domain_start/1, - domains_list/0, + domains_list/0, domains_list/1, domains_list/2, add_port_fwd/2]). -export([init/1, handle_call/3, @@ -26,80 +28,91 @@ -include_lib("khepri/include/khepri.hrl"). -include_lib("khepri/src/khepri_error.hrl"). --define(SERVER, ?MODULE). -define(APPLICATION, virtuerl). --record(state, {server_id, vm_proc_mod, table, idmap}). +-record(state, {cluster, table, idmap, workers}). + + +notify(Pid, Msg) -> + gen_server:call(Pid, {notify, Msg}). create_vm() -> create_vm({default, ?MODULE}). create_vm(Ref) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_create, {default}}). + gen_server:call(?MODULE, {domain_create, {default}}). domain_create(Conf) -> domain_create({default, ?MODULE}, Conf). domain_create(Ref, Conf) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_create, Conf}, infinity). + gen_server:call(?MODULE, {domain_create, Conf}, infinity). domain_delete(Conf) -> domain_delete({default, ?MODULE}, Conf). domain_delete(Ref, Conf) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_delete, Conf}, infinity). + gen_server:call(?MODULE, {domain_delete, Conf}, infinity). domain_get(Conf) -> domain_get({default, ?MODULE}, Conf). domain_get(Ref, Conf) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_get, Conf}). + gen_server:call(?MODULE, {domain_get, Conf}). domains_list() -> domains_list({default, ?MODULE}). +domains_list(Pid) when is_pid(Pid) -> + gen_server:call(Pid, domains_list); domains_list(Ref) -> - gen_server:call({via, virtuerl_reg, Ref}, domains_list). + gen_server:call(?MODULE, domains_list). + + +domains_list(Ref, WorkerName) -> + gen_server:call(?MODULE, {domains_list, WorkerName}). domain_update(Conf) -> domain_update({default, ?MODULE}, Conf). domain_update(Ref, Conf) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_update, Conf}). + gen_server:call(?MODULE, {domain_update, Conf}). domain_stop(Id) -> domain_stop({default, ?MODULE}, Id). +domain_stop(Pid, Id) when is_pid(Pid) -> + gen_server:call(Pid, {domain_update, #{id => Id, state => stopped}}); domain_stop(Ref, Id) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_update, #{id => Id, state => stopped}}). + gen_server:call(?MODULE, {domain_update, #{id => Id, state => stopped}}). domain_start(Id) -> domain_start({default, ?MODULE}, Id). domain_start(Ref, Id) -> - gen_server:call({via, virtuerl_reg, Ref}, {domain_update, #{id => Id, state => running}}). + gen_server:call(?MODULE, {domain_update, #{id => Id, state => running}}). image_from_domain(DomainId, ImageName) -> image_from_domain({default, ?MODULE}, DomainId, ImageName). image_from_domain(Ref, DomainId, ImageName) -> - gen_server:call({via, virtuerl_reg, Ref}, {image_from_domain, #{id => DomainId, image_name => ImageName}}, infinity). + gen_server:call(?MODULE, {image_from_domain, #{id => DomainId, image_name => ImageName}}, infinity). add_port_fwd(DomainId, PortFwd) -> add_port_fwd({default, ?MODULE}, DomainId, PortFwd). add_port_fwd(Ref, DomainId, PortFwd) -> - gen_server:call({via, virtuerl_reg, Ref}, {add_port_fwd, DomainId, PortFwd}). + gen_server:call(?MODULE, {add_port_fwd, DomainId, PortFwd}). -spec domains_list() -> #{}. @@ -115,67 +128,38 @@ home_path() -> application:get_env(?APPLICATION, home, "var"). -start_link(ServerId, Conf) -> - gen_server:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [ServerId, Conf], []). +start(Cluster) -> + gen_server:start({local, ?MODULE}, ?MODULE, [Cluster], []). -init([ServerId, Conf]) -> - #{vm_proc_mod := VmProcMod} = Conf, - net_kernel:monitor_nodes(true), - {ok, #state{server_id = ServerId, vm_proc_mod = VmProcMod, idmap = #{}}, {continue, sync_domains}}. +start_link(Cluster) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Cluster], []). -handle_continue(sync_domains, #state{vm_proc_mod = VmProcMod, idmap = IdMap} = State) -> - {ok, DomainsMap} = khepri:get_many([domain, ?KHEPRI_WILDCARD_STAR]), - Domains = [ {Id, Dom} || #{id := Id} = Dom <- maps:values(DomainsMap) ], - TargetDomains = maps:from_list( - [ {Id, case maps:get(host, Domain, localhost) of localhost -> node(); Else -> Else end} - || {Id, Domain} <- Domains, - case Domain of - #{state := stopped} -> false; - _ -> true - end ]), - AllNodes = nodes([this, visible]), - RunningDomains = maps:from_list(lists:flatten( - [ [ {Id, Node} || {Id, _, _, _} <- supervisor:which_children({virtuerl_sup, Node}), is_binary(Id) ] - || Node <- AllNodes ])), - ToDelete = maps:without(maps:keys(TargetDomains), RunningDomains), - ToAdd = maps:without(maps:keys(RunningDomains), TargetDomains), - [ supervisor:terminate_child({virtuerl_sup, Node}, Id) || {Id, Node} <- maps:to_list(ToDelete) ], - [ supervisor:delete_child({virtuerl_sup, Node}, Id) || {Id, Node} <- maps:to_list(ToDelete) ], - - % lists:foreach(fun () -> ok end, List) - DomsByNode = maps:groups_from_list( - fun({_Id, Dom}) -> case maps:get(host, Dom, localhost) of localhost -> node(); Else -> Else end end, - fun({_Id, Dom}) -> Dom end, - Domains), - [ virtuerl_net:update_net(Node, Doms) || {Node, Doms} <- maps:to_list(DomsByNode), lists:member(Node, AllNodes) ], - - VmPids = [ {Id, - supervisor:start_child({virtuerl_sup, Node}, - {Id, - {VmProcMod, start_link, [maps:get(Id, maps:from_list(Domains))]}, - transient, - infinity, - worker, - []})} || {Id, Node} <- maps:to_list(ToAdd), lists:member(Node, AllNodes) ], - [ virtuerl_pubsub:send({domain_started, DomId}) || {DomId, _} <- VmPids ], - VmPidToDomId = maps:from_list([ {VmPid, DomId} || {DomId, {ok, VmPid}} <- VmPids ]), - [ monitor(process, VmPid) || {_, {ok, VmPid}} <- VmPids ], - - {noreply, State#state{idmap = maps:merge(IdMap, VmPidToDomId)}}. - - -generate_unique_tap_name(TapNames) -> - TapName = io_lib:format("verltap~s", [binary:encode_hex(<<(rand:uniform(16#ffffff)):24>>)]), - case sets:is_element(TapName, TapNames) of - false -> - TapName; - true -> - generate_unique_tap_name(TapNames) - end. +init([Cluster]) -> + pg:monitor(Cluster), + Pids = pg:get_members(Cluster), + [ Pid ! {enslave, self()} || Pid <- Pids ], + {ok, #state{cluster = Cluster, workers = #{}}, {continue, sync_domains}}. + + +handle_continue(sync_domains, #state{cluster = Cluster} = State) -> + Pids = pg:get_members(Cluster), + [ virtuerl_host:sync(Pid) || Pid <- Pids ], + {noreply, State}. +handle_call({notify, Msg}, _From, State) -> + virtuerl_pubsub:send(Msg), + {reply, ok, State}; +handle_call({register_name, Name, Pid}, _From, State) -> + #state{workers = Workers} = State, + {Res, NewState} = case Workers of + #{Name := _} -> {no, State}; + _ -> {yes, State#state{workers = Workers#{Name => Pid}}} + end, + {reply, Res, NewState, {continue, sync_domains}}; + handle_call({domain_create, Conf}, _From, State) -> DomainID = virtuerl_util:uuid4(), Domain0 = maps:merge(#{ @@ -233,8 +217,6 @@ handle_call({domain_create, Conf}, _From, State) -> {ok, DomainsMap} = khepri:get_many([domain, ?KHEPRI_WILDCARD_STAR]), Domains = maps:values(DomainsMap), - TapNames = sets:from_list([ Tap || #{tap_name := Tap} <- Domains ]), - TapName = generate_unique_tap_name(TapNames), % TODO: TapName should be generated on a per-deployment basis <> = <<(rand:uniform(16#ffffffffffff)):48>>, MacAddr = <>, @@ -243,15 +225,14 @@ handle_call({domain_create, Conf}, _From, State) -> mac_addr => MacAddr, ipv4_addr => Ipv4Addr, ipv6_addr => Ipv6Addr, - cidrs => IpCidrs, - tap_name => TapName + cidrs => IpCidrs }, ok = khepri:put([domain, DomainID], DomainWithIps), ?LOG_NOTICE(#{event => domain_ready, domain => DomainWithIps}), virtuerl_pubsub:send({domain_created, DomainID}), {reply, - {ok, maps:merge(#{id => DomainID, tap_name => iolist_to_binary(TapName), mac_addr => binary:encode_hex(MacAddr)}, + {ok, maps:merge(#{id => DomainID, mac_addr => binary:encode_hex(MacAddr)}, maps:map(fun(_, V) -> iolist_to_binary(virtuerl_net:format_ip(V)) end, AddressesMap))}, State, {continue, sync_domains}}; @@ -283,45 +264,40 @@ handle_call(domains_list, _From, State) -> {ok, DomainsMap} = khepri:get_many([domain, ?KHEPRI_WILDCARD_STAR]), Domains = maps:values(DomainsMap), {reply, [ maps:merge(#{host => localhost, state => running, name => Id, vcpu => 1, memory => 512}, Domain) || #{id := Id} = Domain <- Domains ], State}; +handle_call({domains_list, WorkerName}, _From, State) -> + {ok, DomainsMap} = khepri:get_many([domain, ?KHEPRI_WILDCARD_STAR]), + Domains0 = maps:values(DomainsMap), + Domains1 = [ maps:merge(#{host => localhost, state => running, name => Id, vcpu => 1, memory => 512}, Domain) || #{id := Id} = Domain <- Domains0 ], + + FilteredDomains = [ Dom || #{host := Host} = Dom <- Domains1, + case {WorkerName, Host} of + {default, localhost} -> true; + {Name, Name} -> true; + _ -> false + end ], + + {reply, FilteredDomains, State}; handle_call({domain_get, #{id := DomainID}}, _From, State) -> Reply = case khepri:get([domain, DomainID]) of - {ok, #{mac_addr := MacAddr, tap_name := TapName} = Domain} -> + {ok, #{mac_addr := MacAddr} = Domain} -> DomRet = Domain#{ - mac_addr := binary:encode_hex(MacAddr), - tap_name := iolist_to_binary(TapName) + mac_addr := binary:encode_hex(MacAddr) }, {ok, maps:merge(#{host => localhost, state => running, name => DomainID, vcpu => 1, memory => 512}, DomRet)}; _ -> notfound end, {reply, Reply, State}; handle_call({domain_delete, #{id := DomainID}}, _From, State) -> - Res = case khepri:get([domain, DomainID]) of - {ok, Domain} -> - ok = khepri:put([domain, DomainID], Domain#{state => deleting}), - spawn_link(fun() -> - TargetNode = case Domain of - #{host := localhost} -> node(); - #{host := Else} -> Else; - _ -> node() - end, - io:format("terminating ~p~n", [DomainID]), - supervisor:terminate_child({virtuerl_sup, TargetNode}, DomainID), - io:format("done terminating ~p~n", [DomainID]), - supervisor:delete_child({virtuerl_sup, TargetNode}, DomainID), - DomainHomePath = filename:join([virtuerl_mgt:home_path(), "domains", DomainID]), - file:del_dir_r(DomainHomePath), - ok = virtuerl_ipam:unassign(DomainID), - % TODO: we don't really have to delete unused tap here as they will be deleted before the next domain is created - % however, it's of course cleaner to do so here, so consider adding it back - % ok = gen_server:call(virtuerl_net, {net_update}), - ok = khepri:delete([domain, DomainID]), - virtuerl_pubsub:send({domain_deleted, DomainID}) - end), - ok; - _ -> {error, notfound} - - end, - {reply, Res, State}; + case khepri:get([domain, DomainID]) of + {ok, Domain} -> + ok = khepri:put([domain, DomainID], Domain#{state => deleting}), + ok = virtuerl_ipam:unassign(DomainID), % TODO: would be nice to keep networking as long as the node is shutting down (for debugging) + ok = khepri:delete([domain, DomainID]), + virtuerl_pubsub:send({domain_deleted, DomainID}), + {reply, ok, State, {continue, sync_domains}}; + _ -> {reply, {error, notfound}, State} + + end; handle_call({image_from_domain, #{id := DomainID, image_name := ImageName}}, _From, State) -> case khepri:get([domain, DomainID]) of {ok, Domain} -> @@ -355,21 +331,17 @@ handle_cast(_Request, State) -> {noreply, State}. -handle_info({nodedown, _Node}, State) -> +handle_info({_Ref, join, Cluster, Pids}, #state{cluster = Cluster} = State) -> + ?LOG_NOTICE(#{joined => Pids}), + [ Pid ! {enslave, self()} || Pid <- Pids ], {noreply, State}; -handle_info({nodeup, _Node}, State) -> - {noreply, State, {continue, sync_domains}}; -handle_info({'DOWN', _, process, Pid, normal}, #state{idmap = IdMap} = State) -> - NewIdMap = case IdMap of - #{Pid := DomId} -> - {ok, Domain} = khepri:get([domain, DomId]), % TODO: this is empty for a deleted domain - ok = khepri:put([domain, DomId], Domain#{state => stopped}), - maps:remove(Pid, IdMap); - #{} -> - ?LOG_WARNING(#{module => ?MODULE, msg => "process down but not in registry", pid => Pid}), - IdMap - end, - {noreply, State#state{idmap = NewIdMap}}; +handle_info({_Ref, leave, Cluster, Pids}, #state{cluster = Cluster} = State) -> + ?LOG_NOTICE(#{left => Pids}), + #state{workers = Workers} = State, + PidToName = maps:from_list([ {Pid, Name} || {Name, Pid} <- maps:to_list(Workers) ]), + NewWorkers = maps:without(Pids, PidToName), + {noreply, State#state{workers = NewWorkers}}; + handle_info(Info, State) -> ?LOG_NOTICE(#{module => ?MODULE, msg => "unhandled info message", info => Info}), {noreply, State}. diff --git a/apps/virtuerl/src/virtuerl_mgt_sup.erl b/apps/virtuerl/src/virtuerl_mgt_sup.erl new file mode 100644 index 0000000..987dfa7 --- /dev/null +++ b/apps/virtuerl/src/virtuerl_mgt_sup.erl @@ -0,0 +1,27 @@ +-module(virtuerl_mgt_sup). + +-behaviour(supervisor). + +-export([start/1, start_link/1]). +-export([init/1]). + + +start(Cluster) -> + {ok, SupPid} = supervisor:start_child(virtuerl_sup_sup, + #{id => virtuerl_mgt_sup, start => {virtuerl_mgt_sup, start_link, [Cluster]}, type => supervisor}), + {ok, SupPid}. + + +start_link(Cluster) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Cluster]). + + +init([Cluster]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 300, + period => 5 + }, + ChildSpecs = [#{id => virtuerl_ipam, start => {virtuerl_ipam, start_link, []}}, + #{id => virtuerl_mgt, start => {virtuerl_mgt, start_link, [Cluster]}}], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/virtuerl/src/virtuerl_net.erl b/apps/virtuerl/src/virtuerl_net.erl index 0f264f7..0c1271a 100644 --- a/apps/virtuerl/src/virtuerl_net.erl +++ b/apps/virtuerl/src/virtuerl_net.erl @@ -2,31 +2,48 @@ -behaviour(gen_server). --export([start_link/0, update_net/2]). +-export([start_link/2, update_net/2, update_net/3]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -export([parse_cidr/1, format_cidr/1, parse_ip/1, format_ip/1, format_ip_bitstring/1, bridge_addr/1, bridge_addr/2, normalize_net/1]). -include_lib("kernel/include/logger.hrl"). --define(SERVER, ?MODULE). + +-callback get_ifs() -> [If :: #{binary() => binary()}]. + + +-callback run_if_cmds(Cmds :: [Cmd :: term()]) -> ok | {error, Reason :: term()}. + +-behaviour(virtuerl_net). + +%% Callbacks for `virtuerl_net` +-export([get_ifs/0, run_if_cmds/1]). %%%=================================================================== %%% Spawning and gen_server implementation %%%=================================================================== -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +start_link(ServerId, Conf) -> + gen_server:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [Conf], []). update_net(Node, Domains) -> - gen_server:call({?SERVER, Node}, {net_update, Domains}, infinity). + erpc:call(Node, gen_server, call, [{via, virtuerl_reg, {default, ?MODULE}}, {net_update, Domains}, infinity]). -init([]) -> +update_net(Node, Ref, Domains) -> + erpc:call(Node, gen_server, call, [{via, virtuerl_reg, {Ref, ?MODULE}}, {net_update, Domains}, infinity]). + + +-define(NET_PROVIDER, get(net_provider_mod)). + + +init([#{net_prov_mod := NetProviderMod, prefix := IfPrefix}]) -> ?LOG_INFO(#{what => "Started", who => virtuerl_net}), process_flag(trap_exit, true), - {ok, {}}. + put(net_provider_mod, NetProviderMod), + {ok, #{prefix => iolist_to_binary(IfPrefix)}}. terminate(_Reason, _State) -> @@ -38,13 +55,13 @@ terminate(_Reason, _State) -> ok. -handle_call({net_update, Domains}, _From, State) -> - reload_net(Domains), +handle_call({net_update, Domains}, _From, #{prefix := IfPrefix} = State) -> + reload_net(IfPrefix, Domains), {reply, ok, State}. -handle_cast({net_update, Domains}, State) -> - reload_net(Domains), +handle_cast({net_update, Domains}, #{prefix := IfPrefix} = State) -> + reload_net(IfPrefix, Domains), {noreply, State}. @@ -68,7 +85,7 @@ startswith(Str, Pre) -> end. --spec get_cidrs(term) -> {[], binary()}. +-spec get_cidrs(term) -> {[], Ifname :: binary()}. get_cidrs(If) -> Addrs = maps:get(<<"addr_info">>, If, []), Ifname = maps:get(<<"ifname">>, If), @@ -81,20 +98,24 @@ get_cidrs(If) -> end. -reload_net(Domains0) -> +get_ifs() -> + {ok, Ifs} = thoas:decode(os:cmd("ip -j -d addr")), + Ifs. + + +reload_net(IfPrefix, Domains0) -> Domains = [ {Id, Dom} || #{id := Id} = Dom <- Domains0 ], - Output = os:cmd("ip -j addr"), - {ok, JSON} = thoas:decode(Output), + JSON = apply(?NET_PROVIDER, get_ifs, []), % io:format("~p~n", [JSON]), - Matched = maps:from_list([ get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), <<"verlbr">>) ]), + Matched = maps:from_list([ get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), iolist_to_binary([IfPrefix, "br"])) ]), %% lists:foreach(fun(L) -> handle_interface(L, Table) end, Matched), % io:format("Actual: ~p~n", [Matched]), TargetAddrs = sets:from_list([ lists:sort(network_cidrs_to_bride_cidrs(Cidrs)) || {_, #{network_addrs := Cidrs}} <- Domains ]), % io:format("Target: ~p~n", [sets:to_list(TargetAddrs)]), update_nftables(Domains), - sync_taps(Matched, TargetAddrs, Domains), - update_bird_conf(Domains), + sync_taps(IfPrefix, JSON, Matched, TargetAddrs, Domains), + update_bird_conf(IfPrefix, Domains), ok. @@ -268,13 +289,13 @@ format_cidr({<>, Prefixlen}) -> io_lib:format("~s/~B", [format_ip(Addr), Prefixlen]). -generate_unique_bridge_name(Ifnames) -> - Ifname = io_lib:format("verlbr~s", [binary:encode_hex(<<(rand:uniform(16#ffffff)):24>>)]), +generate_unique_bridge_name(IfPrefix, Ifnames) -> + Ifname = iolist_to_binary([IfPrefix, "br", binary:encode_hex(<<(rand:uniform(16#ffffff)):24>>)]), case sets:is_element(Ifname, Ifnames) of false -> Ifname; true -> - generate_unique_bridge_name(Ifnames) + generate_unique_bridge_name(IfPrefix, Ifnames) end. @@ -293,11 +314,10 @@ build_routes([{Addr, Bridge} | L]) -> [io_lib:format(" route ~s via \"~s\";", [format_bird_route(Addr), Bridge]) | build_routes(L)]. -update_bird_conf(Domains) -> - Output = os:cmd("ip -j addr"), - {ok, JSON} = thoas:decode(Output), +update_bird_conf(IfPrefix, Domains) -> + JSON = apply(?NET_PROVIDER, get_ifs, []), - BridgeAddrsToName = [ get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), <<"verlbr">>) ], + BridgeAddrsToName = [ get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), iolist_to_binary([IfPrefix, "br"])) ], BridgeAddrsToNameFlat = maps:from_list([ {parse_cidr(Addr), Name} || {Addrs, Name} <- BridgeAddrsToName, Addr <- Addrs ]), AddrToBridgeName = [ {Addr, maps:get({bridge_addr(Addr, Prefixlen), Prefixlen}, BridgeAddrsToNameFlat)} @@ -317,28 +337,40 @@ reload_bird(Domains) -> ok. --spec sync_taps(#{[term()] => binary()}, term(), term()) -> ok. -sync_taps(ActualAddrs, TargetAddrs, Domains) -> +run_if_cmds(Cmds) -> + User = string:trim(os:cmd("id -un")), + BatchFileContents = [ case Cmd of + {br_del, BrName} -> io_lib:format("link del ~s~n", [BrName]); + {br_add, BrName, Cidrs} -> + AddrAddCmd = [ io_lib:format("addr add ~s dev ~s~n", [Cidr, BrName]) || Cidr <- Cidrs ], + [io_lib:format("link add name ~s type bridge~nlink set ~s up~n", [BrName, BrName]), AddrAddCmd]; + {tap_del, TapName} -> io_lib:format("link del ~s~n", [TapName]); + {tap_add, Tap, Mac, BrName} -> + MacAddrString = virtuerl_util:mac_to_str(Mac), + ["tuntap add dev ", Tap, " mode tap user ", User, "\n", + "link set dev ", Tap, " address ", MacAddrString, " master ", BrName, "\n", + "link set ", Tap, " up\n"] + end || Cmd <- Cmds ], + + run_batch(BatchFileContents). + + +-spec sync_taps(IfPrefix :: binary(), Json :: #{}, #{[term()] => binary()}, term(), term()) -> ok. +sync_taps(IfPrefix, IpRouteJson, ActualAddrs, TargetAddrs, Domains) -> Ifnames = sets:from_list([ Name || {_, Name} <- maps:to_list(ActualAddrs) ]), ToDelete = maps:without(sets:to_list(TargetAddrs), ActualAddrs), ToAdd = sets:subtract(TargetAddrs, sets:from_list(maps:keys(ActualAddrs))), ?LOG_DEBUG("TO DELETE: ~p~n", [ToDelete]), ?LOG_DEBUG("TO ADD: ~p~n", [sets:to_list(ToAdd)]), - BrDeleteCmds = [ io_lib:format("link del ~s~n", [BridgeName]) || BridgeName <- maps:values(ToDelete) ], - BrNameCidrsMap = maps:from_list([ {Cidrs, generate_unique_bridge_name(Ifnames)} || Cidrs <- sets:to_list(ToAdd) ]), - BrAddCmds = maps:values(maps:map(fun(Cidrs, Ifname) -> - AddrAddCmd = [ io_lib:format("addr add ~s dev ~s~n", [Cidr, Ifname]) || Cidr <- Cidrs ], - [io_lib:format("link add name ~s type bridge~nlink set ~s up~n", [Ifname, Ifname]), AddrAddCmd] - end, - BrNameCidrsMap)), + BrDeleteCmds = [ {br_del, BrName} || BrName <- maps:values(ToDelete) ], + BrNameCidrsMap = maps:from_list([ {Cidrs, generate_unique_bridge_name(IfPrefix, Ifnames)} || Cidrs <- sets:to_list(ToAdd) ]), + BrAddCmds = [ {br_add, BrName, Cidrs} || {Cidrs, BrName} <- maps:to_list(BrNameCidrsMap) ], Bridges = maps:merge(maps:with(sets:to_list(TargetAddrs), ActualAddrs), BrNameCidrsMap), - OutputTaps = os:cmd("ip -j link"), - {ok, JSONTaps} = thoas:decode(OutputTaps), % io:format("TAPS: ~p~n", [JSONTaps]), - TapsActual = sets:from_list([ maps:get(<<"ifname">>, L) || L <- JSONTaps, startswith(maps:get(<<"ifname">>, L), <<"verltap">>) ]), - TapsTarget = sets:from_list([ iolist_to_binary(TapName) || {_, #{tap_name := TapName}} <- Domains ]), % TODO: persist tap_name as binary + TapsActual = sets:from_list([ maps:get(<<"ifname">>, L) || L <- IpRouteJson, startswith(maps:get(<<"ifname">>, L), iolist_to_binary([IfPrefix, "tap"])) ]), + TapsTarget = sets:from_list([ iolist_to_binary(TapName) || {_, #{tap_name := TapName}} <- Domains ]), % io:format("Taps Target: ~p~n", [sets:to_list(TapsTarget)]), % io:format("Taps Actual: ~p~n", [sets:to_list(TapsActual)]), TapsMap = maps:from_list([ {iolist_to_binary(Tap), {to_vtap_mac(MacAddr), network_cidrs_to_bride_cidrs(Cidrs)}} @@ -346,24 +378,15 @@ sync_taps(ActualAddrs, TargetAddrs, Domains) -> % io:format("TapsMap: ~p~n", [TapsMap]), TapsToDelete = sets:subtract(TapsActual, TapsTarget), - DeleteCmds = [ io_lib:format("link del ~s~n", [TapName]) || TapName <- sets:to_list(TapsToDelete) ], + DeleteCmds = [ {tap_del, TapName} || TapName <- sets:to_list(TapsToDelete) ], TapsToAdd = sets:subtract(TapsTarget, TapsActual), TapsMapsToAdd = maps:map(fun(_, {MacAddr, Net}) -> {MacAddr, maps:get(Net, Bridges)} end, maps:with(sets:to_list(TapsToAdd), TapsMap)), - User = string:trim(os:cmd("id -un")), - AddCmds = lists:map(fun({Tap, {Mac, Bridge}}) -> - MacAddrString = virtuerl_util:mac_to_str(Mac), - ["tuntap add dev ", Tap, " mode tap user ", User, "\n", - "link set dev ", Tap, " address ", MacAddrString, " master ", Bridge, "\n", - "link set ", Tap, " up\n"] - end, - maps:to_list(TapsMapsToAdd)), + AddCmds = [ {tap_add, Tap, Mac, BrName} || {Tap, {Mac, BrName}} <- maps:to_list(TapsMapsToAdd) ], - BatchFileContents = [BrDeleteCmds, BrAddCmds, DeleteCmds, AddCmds], - - run_batch(BatchFileContents). + apply(?NET_PROVIDER, run_if_cmds, [lists:flatten([DeleteCmds, BrDeleteCmds, BrAddCmds, AddCmds])]). run_nft(IoList) -> run_helper("nft", IoList). diff --git a/apps/virtuerl/src/virtuerl_server.erl b/apps/virtuerl/src/virtuerl_server.erl index 22ed8df..eb3847d 100644 --- a/apps/virtuerl/src/virtuerl_server.erl +++ b/apps/virtuerl/src/virtuerl_server.erl @@ -2,7 +2,7 @@ % -behaviour(gen_server). --export([start/2]). +-export([start/0, start/1, start/2]). %% Callbacks for `gen_server` % -export([init/1, handle_call/3, handle_cast/2]). @@ -21,7 +21,19 @@ % erlang:error(not_implemented). -start(Name, Conf) -> +start() -> start(default). + + +start(Name) -> start(Name, #{}). + + +start(Name, Conf0) -> + Conf = maps:merge(#{ + vm_proc_mod => virtuerl_qemu, + net_prov_mod => virtuerl_net, + prefix => "verl" + }, + Conf0), {ok, SupPid} = supervisor:start_child(virtuerl_sup_sup, #{id => Name, start => {virtuerl_sup, start_link, [Name, Conf]}, type => supervisor}), {ok, SupPid}. diff --git a/apps/virtuerl/src/virtuerl_sup.erl b/apps/virtuerl/src/virtuerl_sup.erl index 344c923..1f2f97c 100644 --- a/apps/virtuerl/src/virtuerl_sup.erl +++ b/apps/virtuerl/src/virtuerl_sup.erl @@ -2,14 +2,28 @@ -behaviour(supervisor). --export([start_link/2]). +-export([start_link/2, start_child/3, which_children/2, delete_child/3, terminate_child/3]). -export([init/1]). --define(SERVER, ?MODULE). + +start_child(ServerId, Node, ChildSpec) -> + erpc:call(Node, supervisor, start_child, [{via, virtuerl_reg, {ServerId, ?MODULE}}, ChildSpec]). + + +which_children(ServerId, Node) -> + erpc:call(Node, supervisor, which_children, [{via, virtuerl_reg, {ServerId, ?MODULE}}]). + + +terminate_child(ServerId, Node, DomId) -> + erpc:call(Node, supervisor, terminate_child, [{via, virtuerl_reg, {ServerId, ?MODULE}}, DomId]). + + +delete_child(ServerId, Node, DomId) -> + erpc:call(Node, supervisor, delete_child, [{via, virtuerl_reg, {ServerId, ?MODULE}}, DomId]). start_link(ServerId, Conf) -> - supervisor:start_link({local, ?SERVER}, ?MODULE, [ServerId, Conf]). + supervisor:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [ServerId, Conf]). init([ServerId, Conf]) -> @@ -18,38 +32,20 @@ init([ServerId, Conf]) -> intensity => 300, period => 5 }, - ChildSpecs = [{virtuerl_pubsub, - {virtuerl_pubsub, start_link, []}, - permanent, - infinity, - worker, - []}, - {virtuerl_ipam, - {virtuerl_ipam, start_link, []}, - permanent, - infinity, - worker, - []}, - {virtuerl_img, - {virtuerl_img, start_link, []}, - permanent, - infinity, - worker, - []}, - {virtuerl_mgt, - {virtuerl_mgt, start_link, [ServerId, Conf]}, + ChildSpecs = [{virtuerl_img, + {virtuerl_img, start_link, [ServerId]}, permanent, infinity, worker, []}, {virtuerl_net, - {virtuerl_net, start_link, []}, + {virtuerl_net, start_link, [ServerId, Conf]}, permanent, infinity, worker, []}, - {virtuerl_api, - {virtuerl_api, start_link, []}, + {virtuerl_host, + {virtuerl_host, start_link, [ServerId, Conf]}, permanent, infinity, worker, diff --git a/apps/virtuerl/src/virtuerl_sup_sup.erl b/apps/virtuerl/src/virtuerl_sup_sup.erl index cd6ff7b..145d3b3 100644 --- a/apps/virtuerl/src/virtuerl_sup_sup.erl +++ b/apps/virtuerl/src/virtuerl_sup_sup.erl @@ -16,5 +16,7 @@ init([]) -> intensity => 300, period => 5 }, - ChildSpecs = [#{id => virtuerl_reg, start => {virtuerl_reg, start_link, []}}], + ChildSpecs = [#{id => virtuerl_reg, start => {virtuerl_reg, start_link, []}}, + #{id => virtuerl_pubsub, start => {virtuerl_pubsub, start_link, []}}, + #{id => pg, start => {pg, start_link, []}}], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/virtuerl/test/virtuerl_SUITE.erl b/apps/virtuerl/test/virtuerl_SUITE.erl index f7fe86f..d92e925 100644 --- a/apps/virtuerl/test/virtuerl_SUITE.erl +++ b/apps/virtuerl/test/virtuerl_SUITE.erl @@ -11,6 +11,8 @@ all() -> [test_create_domain, test_create_domain_dualstack]. init_per_suite(Config) -> {ok, _} = application:ensure_all_started(virtuerl), + virtuerl_server:start(default, #{cluster => default}), + virtuerl_mgt_sup:start(default), Config. diff --git a/apps/virtuerl/test/virtuerl_mock_SUITE.erl b/apps/virtuerl/test/virtuerl_mock_SUITE.erl index 88a29f1..e01a589 100644 --- a/apps/virtuerl/test/virtuerl_mock_SUITE.erl +++ b/apps/virtuerl/test/virtuerl_mock_SUITE.erl @@ -11,7 +11,10 @@ all() -> [test_create_domain]. init_per_suite(Config) -> {ok, _} = application:ensure_all_started(virtuerl), - virtuerl_server:start(default, #{vm_proc_mod => virtuerl_mock_vm}), + virtuerl_mock_net:start(), + virtuerl_server:start(default, #{cluster => default, vm_proc_mod => virtuerl_mock_vm, net_prov_mod => virtuerl_mock_net, prefix => "verl0"}), + virtuerl_server:start(test2, #{cluster => default, vm_proc_mod => virtuerl_mock_vm, net_prov_mod => virtuerl_mock_net, prefix => "verl1"}), + virtuerl_mgt_sup:start(default), Config. @@ -61,17 +64,34 @@ runcmd: {domain_started, DomId} -> ok end, + ct:print("Before delete ~p~n", [virtuerl_mock_net:get_ifs()]), + virtuerl_mgt:domain_delete(#{id => DomId}), % make sure address is actually released and reused receive {domain_deleted, DomId} -> ok + after + 1000 -> + erlang:error("timed out waiting for domain_deleted DomId") end, + {ok, #{id := Dom2Id, ipv4_addr := <<"192.168.17.8">>}} = virtuerl_mgt:domain_create(#{name => "test_domain_2", vcpu => 1, memory => 512, network_id => NetID, user_data => ""}), + receive + {domain_started, Dom2Id} -> ok + after + 1000 -> + erlang:error("timed out waiting for domain_started Dom2Id") + end, virtuerl_mgt:domain_delete(#{id => Dom2Id}), receive - {domain_deleted, Dom2Id} -> ok + {domain_stopped, Dom2Id} -> ok + after + 1000 -> + erlang:error("timed out waiting for domain_stopped Dom2Id") end, + ct:print("After stopped ~p~n", [virtuerl_mock_net:get_ifs()]), + [] = virtuerl_mock_net:get_ifs(), virtuerl_ipam:ipam_delete_net(NetID), diff --git a/apps/virtuerl/test/virtuerl_mock_net.erl b/apps/virtuerl/test/virtuerl_mock_net.erl new file mode 100644 index 0000000..37249d2 --- /dev/null +++ b/apps/virtuerl/test/virtuerl_mock_net.erl @@ -0,0 +1,72 @@ +-module(virtuerl_mock_net). + +-behaviour(gen_server). +-behaviour(virtuerl_net). + +-export([start/0, start_link/0]). + +%% Callbacks for `gen_server` +-export([init/1, handle_call/3, handle_cast/2]). + +%% Callbacks for `virtuerl_net` +-export([get_ifs/0, run_if_cmds/1]). + + +get_ifs() -> + gen_server:call(?MODULE, get_ifs). + + +run_if_cmds(Cmds) -> + gen_server:call(?MODULE, {run_if_cmds, Cmds}). + + +start() -> + gen_server:start({local, ?MODULE}, ?MODULE, [], []). + + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + + +init([]) -> + {ok, #{}}. + + +handle_call(get_ifs, _From, State) -> + {reply, maps:values(State), State}; +handle_call({run_if_cmds, Cmds}, _From, State) -> + ct:print("~p~n", [Cmds]), + NewState = lists:foldl(fun(Cmd, Acc) -> + case Cmd of + {br_del, BrName} -> + {ok, _} = maps:find(BrName, Acc), + maps:remove(BrName, Acc); + {tap_del, TapName} -> + {ok, _} = maps:find(TapName, Acc), + maps:remove(TapName, Acc); + {br_add, BrName, Cidrs} -> + error = maps:find(BrName, Acc), + maps:put(BrName, + #{ + <<"ifname">> => BrName, + <<"addr_info">> => + lists:map(fun(Cidr) -> + {Addr, Prefixlen} = virtuerl_net:parse_cidr(Cidr), + #{<<"local">> => virtuerl_net:format_ip_bitstring(Addr), <<"prefixlen">> => Prefixlen, <<"scope">> => <<"global">>} + end, + Cidrs) + }, + Acc); + {tap_add, Tap, _Mac, _BrName} -> + error = maps:find(Tap, Acc), + maps:put(Tap, #{<<"ifname">> => Tap}, Acc) + end + end, + State, + Cmds), + + {reply, ok, NewState}. + + +handle_cast(Request, State) -> + erlang:error(not_implemented). diff --git a/apps/virtuerl/test/virtuerl_nested_SUITE.erl b/apps/virtuerl/test/virtuerl_nested_SUITE.erl index 7870703..5883d8e 100644 --- a/apps/virtuerl/test/virtuerl_nested_SUITE.erl +++ b/apps/virtuerl/test/virtuerl_nested_SUITE.erl @@ -13,6 +13,8 @@ all() -> [test_create_domain]. init_per_suite(Config) -> {ok, _} = application:ensure_all_started(virtuerl), + virtuerl_server:start(default, #{cluster => default}), + virtuerl_mgt_sup:start(default), Config. diff --git a/config/sys.config b/config/sys.config index 95812d0..006e565 100644 --- a/config/sys.config +++ b/config/sys.config @@ -1,15 +1,28 @@ [{kernel, [{logger_level, all}, - {logger, - [{handler, default, logger_std_h, - #{ level => info, - formatter => {logger_formatter, #{single_line => false}}}}, - {handler, debug, logger_disk_log_h, - #{ level => debug, - config => #{file => "var/log/virtuerl.log"}, - max_no_bytes => 10485760, - formatter => {logger_formatter, #{single_line => false}}}} - ]}]}, - {erlexec, []}, - {virtuerl, [{servers, #{default => #{vm_proc_mod => virtuerl_qemu}}}]} -]. + {logger, + [{handler, default, + logger_std_h, + #{ + level => info, + formatter => {logger_formatter, #{single_line => false}} + }}, + {handler, debug, + logger_disk_log_h, + #{ + level => debug, + config => #{file => "var/log/virtuerl.log"}, + max_no_bytes => 10485760, + formatter => {logger_formatter, #{single_line => false}} + }}]}]}, + {erlexec, []}, + {virtuerl, [{clusters, [default]}, + {servers, + #{ + default => #{ + cluster => default, + vm_proc_mod => virtuerl_qemu, + net_prov_mod => virtuerl_net, + prefix => "verlbr" + } + }}]}]. diff --git a/config/test.sys.config b/config/test.sys.config index 2e999e3..1371b73 100644 --- a/config/test.sys.config +++ b/config/test.sys.config @@ -1,14 +1,10 @@ [{kernel, [{logger_level, all}, - {logger, - [{handler, default, logger_std_h, - #{ level => info, - formatter => {logger_formatter, #{single_line => false}}}}, - {handler, debug, logger_disk_log_h, - #{ level => debug, - config => #{file => "var/log/virtuerl.log"}, - max_no_bytes => 10485760, - formatter => {logger_formatter, #{single_line => false}}}} - ]}]}, - {erlexec, []} -]. + {logger, + [{handler, default, + logger_std_h, + #{ + level => debug, + formatter => {logger_formatter, #{single_line => false}} + }}]}]}, + {erlexec, []}].