From 1c3512b82528a3009c5a2939c4d20a809a80a214 Mon Sep 17 00:00:00 2001 From: Ilya Verbitskiy Date: Tue, 27 Aug 2024 16:31:54 +0200 Subject: [PATCH] refactor and decouple QEMU for better testing --- apps/virtuerl/src/virtuerl_app.erl | 10 ++- apps/virtuerl/src/virtuerl_mgt.erl | 97 +++++++++++++++++--------- apps/virtuerl/src/virtuerl_reg.erl | 57 +++++++++++++++ apps/virtuerl/src/virtuerl_server.erl | 27 +++++++ apps/virtuerl/src/virtuerl_sup.erl | 10 +-- apps/virtuerl/src/virtuerl_sup_sup.erl | 20 ++++++ config/sys.config | 3 +- config/test.sys.config | 14 ++++ rebar.config | 2 +- test/virtuerl_mock_SUITE.erl | 97 ++++++++++++++++++++++++++ test/virtuerl_mock_vm.erl | 25 +++++++ 11 files changed, 322 insertions(+), 40 deletions(-) create mode 100644 apps/virtuerl/src/virtuerl_reg.erl create mode 100644 apps/virtuerl/src/virtuerl_server.erl create mode 100644 apps/virtuerl/src/virtuerl_sup_sup.erl create mode 100644 config/test.sys.config create mode 100644 test/virtuerl_mock_SUITE.erl create mode 100644 test/virtuerl_mock_vm.erl diff --git a/apps/virtuerl/src/virtuerl_app.erl b/apps/virtuerl/src/virtuerl_app.erl index ea7b4f6..b48fbf2 100644 --- a/apps/virtuerl/src/virtuerl_app.erl +++ b/apps/virtuerl/src/virtuerl_app.erl @@ -19,7 +19,15 @@ start(_StartType, _StartArgs) -> ok end, - virtuerl_sup:start_link(). + {ok, Pid} = virtuerl_sup_sup:start_link(), + + Servers = case application:get_env(servers) of + undefined -> #{}; + {ok, Servers0} -> Servers0 + end, + [ virtuerl_server:start(Name, Conf) || {Name, Conf} <- maps:to_list(Servers) ], + + {ok, Pid}. start() -> diff --git a/apps/virtuerl/src/virtuerl_mgt.erl b/apps/virtuerl/src/virtuerl_mgt.erl index ff6cdcf..7e87f26 100644 --- a/apps/virtuerl/src/virtuerl_mgt.erl +++ b/apps/virtuerl/src/virtuerl_mgt.erl @@ -2,7 +2,7 @@ -behaviour(gen_server). --export([start_link/0, +-export([start_link/2, home_path/0, image_from_domain/2, domain_update/1, @@ -29,71 +29,103 @@ -define(SERVER, ?MODULE). -define(APPLICATION, virtuerl). --record(state, {table, idmap}). +-record(state, {server_id, vm_proc_mod, table, idmap}). -create_vm() -> - gen_server:call(?SERVER, {domain_create, {default}}). +create_vm() -> create_vm({default, ?MODULE}). -%%create_vm(#{cpus := NumCPUs, memory := Memory}) -> -domain_create(Conf) -> - gen_server:call(?SERVER, {domain_create, Conf}, infinity). +create_vm(Ref) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_create, {default}}). -domain_delete(Conf) -> - gen_server:call(?SERVER, {domain_delete, Conf}, infinity). +domain_create(Conf) -> domain_create({default, ?MODULE}, Conf). -domain_get(Conf) -> - gen_server:call(?SERVER, {domain_get, Conf}). +domain_create(Ref, Conf) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_create, Conf}, infinity). --spec domains_list() -> #{}. -domains_list() -> - gen_server:call(?SERVER, domains_list). +domain_delete(Conf) -> domain_delete({default, ?MODULE}, Conf). -domain_update(Conf) -> - gen_server:call(?SERVER, {domain_update, Conf}). +domain_delete(Ref, Conf) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_delete, Conf}, infinity). -domain_stop(Id) -> - gen_server:call(?SERVER, {domain_update, #{id => Id, state => stopped}}). +domain_get(Conf) -> domain_get({default, ?MODULE}, Conf). -domain_start(Id) -> - gen_server:call(?SERVER, {domain_update, #{id => Id, state => running}}). +domain_get(Ref, Conf) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_get, Conf}). -image_from_domain(DomainId, ImageName) -> - gen_server:call(?SERVER, {image_from_domain, #{id => DomainId, image_name => ImageName}}, infinity). +domains_list() -> domains_list({default, ?MODULE}). --spec add_port_fwd(binary(), #{protos := [tcp | udp], source_port := integer(), target_port := integer()}) -> term(). -add_port_fwd(DomainId, PortFwd) -> - gen_server:call(?SERVER, {add_port_fwd, DomainId, PortFwd}). +domains_list(Ref) -> + gen_server:call({via, virtuerl_reg, Ref}, domains_list). + + +domain_update(Conf) -> domain_update({default, ?MODULE}, Conf). + + +domain_update(Ref, Conf) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_update, Conf}). + + +domain_stop(Id) -> domain_stop({default, ?MODULE}, Id). + + +domain_stop(Ref, Id) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_update, #{id => Id, state => stopped}}). + + +domain_start(Id) -> domain_start({default, ?MODULE}, Id). + + +domain_start(Ref, Id) -> + gen_server:call({via, virtuerl_reg, Ref}, {domain_update, #{id => Id, state => running}}). +image_from_domain(DomainId, ImageName) -> image_from_domain({default, ?MODULE}, DomainId, ImageName). + + +image_from_domain(Ref, DomainId, ImageName) -> + gen_server:call({via, virtuerl_reg, Ref}, {image_from_domain, #{id => DomainId, image_name => ImageName}}, infinity). + + +add_port_fwd(DomainId, PortFwd) -> add_port_fwd({default, ?MODULE}, DomainId, PortFwd). + + +add_port_fwd(Ref, DomainId, PortFwd) -> + gen_server:call({via, virtuerl_reg, Ref}, {add_port_fwd, DomainId, PortFwd}). + + +-spec domains_list() -> #{}. + + +-spec add_port_fwd(binary(), #{protos := [tcp | udp], source_port := integer(), target_port := integer()}) -> term(). + %%%=================================================================== %%% Spawning and gen_server implementation %%%=================================================================== - home_path() -> application:get_env(?APPLICATION, home, "var"). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +start_link(ServerId, Conf) -> + gen_server:start_link({via, virtuerl_reg, {ServerId, ?MODULE}}, ?MODULE, [ServerId, Conf], []). -init([]) -> +init([ServerId, Conf]) -> + #{vm_proc_mod := VmProcMod} = Conf, net_kernel:monitor_nodes(true), - {ok, #state{idmap = #{}}, {continue, sync_domains}}. + {ok, #state{server_id = ServerId, vm_proc_mod = VmProcMod, idmap = #{}}, {continue, sync_domains}}. -handle_continue(sync_domains, #state{idmap = IdMap} = State) -> +handle_continue(sync_domains, #state{vm_proc_mod = VmProcMod, idmap = IdMap} = State) -> {ok, DomainsMap} = khepri:get_many([domain, ?KHEPRI_WILDCARD_STAR]), Domains = [ {Id, Dom} || #{id := Id} = Dom <- maps:values(DomainsMap) ], TargetDomains = maps:from_list( @@ -122,11 +154,12 @@ handle_continue(sync_domains, #state{idmap = IdMap} = State) -> VmPids = [ {Id, supervisor:start_child({virtuerl_sup, Node}, {Id, - {virtuerl_qemu, start_link, [maps:get(Id, maps:from_list(Domains))]}, + {VmProcMod, start_link, [maps:get(Id, maps:from_list(Domains))]}, transient, infinity, worker, []})} || {Id, Node} <- maps:to_list(ToAdd), lists:member(Node, AllNodes) ], + [ virtuerl_pubsub:send({domain_started, DomId}) || {DomId, _} <- VmPids ], VmPidToDomId = maps:from_list([ {VmPid, DomId} || {DomId, {ok, VmPid}} <- VmPids ]), [ monitor(process, VmPid) || {_, {ok, VmPid}} <- VmPids ], diff --git a/apps/virtuerl/src/virtuerl_reg.erl b/apps/virtuerl/src/virtuerl_reg.erl new file mode 100644 index 0000000..e2bd481 --- /dev/null +++ b/apps/virtuerl/src/virtuerl_reg.erl @@ -0,0 +1,57 @@ +-module(virtuerl_reg). + +-behaviour(gen_server). + +-export([start_link/0, register_name/2, unregister_name/1, whereis_name/1, send/2]). +%% Callbacks for `gen_server` +-export([init/1, handle_call/3, handle_cast/2]). + +-type name() :: {atom(), atom()}. + + +-spec register_name(Name :: name(), Pid :: pid()) -> yes | no. +register_name(Name, Pid) -> gen_server:call(?MODULE, {register_name, Name, Pid}). + + +-spec unregister_name(Name :: name()) -> _. +unregister_name(Name) -> gen_server:cast(?MODULE, {unregister_name, Name}). + + +-spec whereis_name(Name :: name()) -> pid() | undefined. +whereis_name(Name) -> gen_server:call(?MODULE, {whereis_name, Name}). + + +-spec send(Name :: name(), Msg :: term()) -> pid(). +send(Name, Msg) -> + case whereis_name(Name) of + undefined -> exit({badarg, Name, Msg}); + Pid -> Pid ! Msg, Pid + end. + + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + + +init([]) -> + {ok, #{}}. + + +% -spec handle_call({register_name, name(), pid()}, _From :: _, State :: #{}) -> {reply, no | yes, #{}}. +handle_call({register_name, Name, Pid}, _From, State) -> + {Res, NewState} = case State of + #{Name := _} -> {no, State}; + _ -> {yes, State#{Name => Pid}} + end, + {reply, Res, NewState}; + +handle_call({whereis_name, Name}, _From, State) -> + Res = case State of + #{Name := Pid} -> Pid; + _ -> undefined + end, + {reply, Res, State}. + + +handle_cast({unregister_name, Name}, State) -> + {noreply, ok, maps:remove(Name, State)}. diff --git a/apps/virtuerl/src/virtuerl_server.erl b/apps/virtuerl/src/virtuerl_server.erl new file mode 100644 index 0000000..22ed8df --- /dev/null +++ b/apps/virtuerl/src/virtuerl_server.erl @@ -0,0 +1,27 @@ +-module(virtuerl_server). + +% -behaviour(gen_server). + +-export([start/2]). + +%% Callbacks for `gen_server` +% -export([init/1, handle_call/3, handle_cast/2]). + +% start_link([]) -> +% gen_server:start_link({local, ?MODULE}, ?MODULE, []). + +% init([]) -> +% _ = ets:new(virtuerl_servers, [named_table]), +% Tab = ets:whereis(virtuerl_servers), +% erlang:error(not_implemented). + +% handle_call({update_config, Name, Conf},_From,State) -> + +% handle_cast(Request,State) -> +% erlang:error(not_implemented). + + +start(Name, Conf) -> + {ok, SupPid} = supervisor:start_child(virtuerl_sup_sup, + #{id => Name, start => {virtuerl_sup, start_link, [Name, Conf]}, type => supervisor}), + {ok, SupPid}. diff --git a/apps/virtuerl/src/virtuerl_sup.erl b/apps/virtuerl/src/virtuerl_sup.erl index 4357b80..344c923 100644 --- a/apps/virtuerl/src/virtuerl_sup.erl +++ b/apps/virtuerl/src/virtuerl_sup.erl @@ -2,17 +2,17 @@ -behaviour(supervisor). --export([start_link/0]). +-export([start_link/2]). -export([init/1]). -define(SERVER, ?MODULE). -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). +start_link(ServerId, Conf) -> + supervisor:start_link({local, ?SERVER}, ?MODULE, [ServerId, Conf]). -init([]) -> +init([ServerId, Conf]) -> SupFlags = #{ strategy => one_for_one, intensity => 300, @@ -37,7 +37,7 @@ init([]) -> worker, []}, {virtuerl_mgt, - {virtuerl_mgt, start_link, []}, + {virtuerl_mgt, start_link, [ServerId, Conf]}, permanent, infinity, worker, diff --git a/apps/virtuerl/src/virtuerl_sup_sup.erl b/apps/virtuerl/src/virtuerl_sup_sup.erl new file mode 100644 index 0000000..cd6ff7b --- /dev/null +++ b/apps/virtuerl/src/virtuerl_sup_sup.erl @@ -0,0 +1,20 @@ +-module(virtuerl_sup_sup). + +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). + + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + + +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 300, + period => 5 + }, + ChildSpecs = [#{id => virtuerl_reg, start => {virtuerl_reg, start_link, []}}], + {ok, {SupFlags, ChildSpecs}}. diff --git a/config/sys.config b/config/sys.config index 2e999e3..95812d0 100644 --- a/config/sys.config +++ b/config/sys.config @@ -10,5 +10,6 @@ max_no_bytes => 10485760, formatter => {logger_formatter, #{single_line => false}}}} ]}]}, - {erlexec, []} + {erlexec, []}, + {virtuerl, [{servers, #{default => #{vm_proc_mod => virtuerl_qemu}}}]} ]. diff --git a/config/test.sys.config b/config/test.sys.config new file mode 100644 index 0000000..2e999e3 --- /dev/null +++ b/config/test.sys.config @@ -0,0 +1,14 @@ +[{kernel, + [{logger_level, all}, + {logger, + [{handler, default, logger_std_h, + #{ level => info, + formatter => {logger_formatter, #{single_line => false}}}}, + {handler, debug, logger_disk_log_h, + #{ level => debug, + config => #{file => "var/log/virtuerl.log"}, + max_no_bytes => 10485760, + formatter => {logger_formatter, #{single_line => false}}}} + ]}]}, + {erlexec, []} +]. diff --git a/rebar.config b/rebar.config index cc0290a..4c1d489 100644 --- a/rebar.config +++ b/rebar.config @@ -11,7 +11,7 @@ {shell, [{config, "config/sys.config"}, {apps, [virtuerl, virtuerl_ui]}]}. -{ct_opts, [{sys_config, ["config/sys.config"]}]}. +{ct_opts, [{sys_config, ["config/test.sys.config"]}]}. {project_plugins, [erlfmt, rebar3_efmt, diff --git a/test/virtuerl_mock_SUITE.erl b/test/virtuerl_mock_SUITE.erl new file mode 100644 index 0000000..88a29f1 --- /dev/null +++ b/test/virtuerl_mock_SUITE.erl @@ -0,0 +1,97 @@ +-module(virtuerl_mock_SUITE). + +-behaviour(ct_suite). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([test_create_domain/1]). + + +all() -> [test_create_domain]. + + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(virtuerl), + virtuerl_server:start(default, #{vm_proc_mod => virtuerl_mock_vm}), + Config. + + +end_per_suite(_Config) -> + application:stop(virtuerl). + + +test_create_domain(_Config) -> + virtuerl_pubsub:subscribe(), + + {ok, NetID} = virtuerl_ipam:ipam_create_net([{<<192:8, 168:8, 17:8, 0:8>>, 24}]), + {ok, #{id := DomId, ipv4_addr := <<"192.168.17.8">>}} = + virtuerl_mgt:domain_create( + #{ + name => "test_domain", + vcpu => 1, + memory => 512, + network_id => NetID, + inbound_rules => [#{protocols => ["tcp"], target_ports => [80]}], + user_data => + "#cloud-config + +users: + - name: tester + passwd: $6$Cf1HnaIWk8TunKFs$40sITB7utYJbVL9kkmhVwzCW33vbq55IGSbpLp1AqOufng1qNxf8wyHj4fdp3xMAfr0yrGioiWwtRvbN58rlI. + lock_passwd: false + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBDCT3LrJenezXzP9T6519IgpVCP1uv6f5iQwZ+IDdFc + +apt: + primary: + - arches: [default] + search: + - http://mirror.ipb.de/debian/ + - http://security.debian.org/debian-security + +packages: + - nginx + +runcmd: + - service nginx restart +" + }), % password: asd + receive + {domain_started, DomId} -> ok + end, + + virtuerl_mgt:domain_delete(#{id => DomId}), + % make sure address is actually released and reused + receive + {domain_deleted, DomId} -> ok + end, + {ok, #{id := Dom2Id, ipv4_addr := <<"192.168.17.8">>}} = + virtuerl_mgt:domain_create(#{name => "test_domain_2", vcpu => 1, memory => 512, network_id => NetID, user_data => ""}), + virtuerl_mgt:domain_delete(#{id => Dom2Id}), + receive + {domain_deleted, Dom2Id} -> ok + end, + + virtuerl_ipam:ipam_delete_net(NetID), + + ok. + + +wait_for_http(Url, Timeout) -> + Deadline = erlang:system_time(millisecond) + Timeout, + do_wait_for_http(Url, Deadline). + + +do_wait_for_http(Url, Deadline) -> + case httpc:request(get, {Url, []}, [{timeout, 1000}], []) of + {error, _} -> + case erlang:system_time(millisecond) > Deadline of + true -> + {error, timeout}; + false -> + timer:sleep(2000), + do_wait_for_http(Url, Deadline) + end; + Res -> Res + end. diff --git a/test/virtuerl_mock_vm.erl b/test/virtuerl_mock_vm.erl new file mode 100644 index 0000000..478ebe1 --- /dev/null +++ b/test/virtuerl_mock_vm.erl @@ -0,0 +1,25 @@ +-module(virtuerl_mock_vm). + +-behaviour(gen_server). + +-export([start_link/1]). + +%% Callbacks for `gen_server` +-export([init/1, handle_call/3, handle_cast/2]). + + +start_link(Dom) -> + gen_server:start_link(?MODULE, [Dom], []). + + +init([Dom]) -> + io:format("MOCK/VM: ~p~n", [Dom]), + {ok, {}}. + + +handle_call(Request, From, State) -> + erlang:error(not_implemented). + + +handle_cast(Request, State) -> + erlang:error(not_implemented).