From 23f5dfec23a8be3d92aa74751992f774a2edee2b Mon Sep 17 00:00:00 2001 From: Ilya Verbitskiy Date: Fri, 5 Jan 2024 23:29:08 +0100 Subject: [PATCH] virtuerl: wip --- .github/workflows/build.yaml | 6 +- virtuerl/.idea/modules.xml | 1 + virtuerl/config/sys.config | 5 +- virtuerl/rebar.config | 8 +- virtuerl/rebar.lock | 3 + virtuerl/src/virtuerl.app.src | 6 +- virtuerl/src/virtuerl_app.erl | 2 + virtuerl/src/virtuerl_ipam.erl | 10 +- virtuerl/src/virtuerl_mgt.erl | 75 +++++++++--- virtuerl/src/virtuerl_net.erl | 25 ++-- virtuerl/src/virtuerl_qemu.erl | 211 ++++++++++++++++++++++++++------- virtuerl/src/virtuerl_qmp.erl | 97 +++++++++++++++ virtuerl/src/virtuerl_stor.erl | 48 ++++++++ virtuerl/src/virtuerl_util.erl | 13 +- 14 files changed, 424 insertions(+), 86 deletions(-) create mode 100644 virtuerl/src/virtuerl_qmp.erl create mode 100644 virtuerl/src/virtuerl_stor.erl diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2a1fb53..4656183 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: - name: Setup Virtuerl working-directory: virtuerl run: | - sudo -s rebar3 compile + rebar3 release cat << EOF | sudo tee -a /etc/systemd/system/virtuerl.service [Unit] @@ -47,8 +47,8 @@ jobs: After=network.target [Service] - WorkingDirectory=${PWD} - ExecStart=/bin/sh -c 'erl -pa _build/default/lib/*/ebin -config config/sys.config -s virtuerl_app -noshell -noinput' + WorkingDirectory=${PWD}/_build/default/rel/virtuerl/ + ExecStart=bin/virtuerl foreground Restart=always [Install] diff --git a/virtuerl/.idea/modules.xml b/virtuerl/.idea/modules.xml index 540417d..d627cff 100644 --- a/virtuerl/.idea/modules.xml +++ b/virtuerl/.idea/modules.xml @@ -5,6 +5,7 @@ + diff --git a/virtuerl/config/sys.config b/virtuerl/config/sys.config index 9cd128b..c65080a 100644 --- a/virtuerl/config/sys.config +++ b/virtuerl/config/sys.config @@ -2,7 +2,7 @@ [{logger_level, all}, {logger, [{handler, default, logger_std_h, - #{ level => info, + #{ level => debug, formatter => {logger_formatter, #{single_line => false}}}} ]}]}, {grpcbox, [ @@ -11,5 +11,6 @@ {keyfile, "config/client.key"}, {cacertfile, "config/ca.crt"} ]}], #{}}]}} - ]} + ]}, + {erlexec, [{root, true}, {user, "root"}]} ]. diff --git a/virtuerl/rebar.config b/virtuerl/rebar.config index 0ec98a0..c3f9abc 100644 --- a/virtuerl/rebar.config +++ b/virtuerl/rebar.config @@ -4,7 +4,13 @@ {thoas, "1.0.0"}, {grpcbox, "0.16.0"}, {cowboy, "2.10.0"}, - {mochiweb, "3.1.2"} + {mochiweb, "3.1.2"}, + {erlexec, "~> 2.0"} +]}. + +{relx, [ + {release, {virtuerl, git}, [virtuerl, {khepri, load}, {mnesia, load}]}, + {mode, prod} ]}. {plugins, [grpcbox_plugin]}. diff --git a/virtuerl/rebar.lock b/virtuerl/rebar.lock index 21dace2..347e593 100644 --- a/virtuerl/rebar.lock +++ b/virtuerl/rebar.lock @@ -5,6 +5,7 @@ {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.10.0">>},0}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.12.1">>},1}, {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1}, + {<<"erlexec">>,{pkg,<<"erlexec">>,<<"2.0.2">>},0}, {<<"gen_batch_server">>,{pkg,<<"gen_batch_server">>,<<"0.8.8">>},2}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1}, {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},0}, @@ -24,6 +25,7 @@ {<<"cowboy">>, <<"FF9FFEFF91DAE4AE270DD975642997AFE2A1179D94B1887863E43F681A203E26">>}, {<<"cowlib">>, <<"A9FA9A625F1D2025FE6B462CB865881329B5CAFF8F1854D1CBC9F9533F00E1E1">>}, {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>}, + {<<"erlexec">>, <<"995E40477DE94C37EC1264CC3E52EB6273938E80C9BCC4F94110A3F1C0D9ABA3">>}, {<<"gen_batch_server">>, <<"7840A1FA63EE1EFFC83E8A91D22664847A2BA1192D30EAFFFD914ACB51578068">>}, {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>}, {<<"grpcbox">>, <<"B83F37C62D6EECA347B77F9B1EC7E9F62231690CDFEB3A31BE07CD4002BA9C82">>}, @@ -42,6 +44,7 @@ {<<"cowboy">>, <<"3AFDCCB7183CC6F143CB14D3CF51FA00E53DB9EC80CDCD525482F5E99BC41D6B">>}, {<<"cowlib">>, <<"163B73F6367A7341B33C794C4E88E7DBFE6498AC42DCD69EF44C5BC5507C8DB0">>}, {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>}, + {<<"erlexec">>, <<"CC829A7C6C23D399832DA2E998EA5EBC552232A6FE3EB1EDB400178EC8287DCB">>}, {<<"gen_batch_server">>, <<"C3E6A1A2A0FB62AEE631A98CFA0FD8903E9562422CBF72043953E2FB1D203017">>}, {<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>}, {<<"grpcbox">>, <<"294DF743AE20A7E030889F00644001370A4F7CE0121F3BBDAF13CF3169C62913">>}, diff --git a/virtuerl/src/virtuerl.app.src b/virtuerl/src/virtuerl.app.src index 556bc45..8d29911 100644 --- a/virtuerl/src/virtuerl.app.src +++ b/virtuerl/src/virtuerl.app.src @@ -6,7 +6,11 @@ {applications, [kernel, stdlib, - cowboy + inets, + mochiweb, + cowboy, + thoas, + erlexec ]}, {env,[]}, {modules, []}, diff --git a/virtuerl/src/virtuerl_app.erl b/virtuerl/src/virtuerl_app.erl index db57f77..6a3d662 100644 --- a/virtuerl/src/virtuerl_app.erl +++ b/virtuerl/src/virtuerl_app.erl @@ -19,10 +19,12 @@ start(_StartType, _StartArgs) -> %% Res = cowboy:start_clear(my_listener, [{port, 8080}], #{env => #{dispatch => Dispatch}}), %% io:format("RESULT: ~p~n", [Res]), %% {ok, _} = Res, +%% exec:debug(4), virtuerl_sup:start_link(). start() -> application:ensure_all_started(virtuerl). +%% exec:debug(4). stop(_State) -> ok. diff --git a/virtuerl/src/virtuerl_ipam.erl b/virtuerl/src/virtuerl_ipam.erl index 54aa24b..a839199 100644 --- a/virtuerl/src/virtuerl_ipam.erl +++ b/virtuerl/src/virtuerl_ipam.erl @@ -96,19 +96,13 @@ start_link() -> init([]) -> io:format("starting IPAM service~n"), - {ok, StoreId} = khepri:start(), + {ok, StoreId} = khepri:start(filename:join(virtuerl_mgt:home_path(), "khepri")), init([StoreId]); init([StoreId]) -> {ok, StoreId}. terminate(_Reason, StoreId) -> - DefaultStoreId = khepri_cluster:get_default_store_id(), - case StoreId of - DefaultStoreId -> - khepri:stop(StoreId); - _ -> - ok - end. + khepri:stop(StoreId). handle_call(net_list, _From, StoreId) -> case khepri:get_many(StoreId, [network, ?KHEPRI_WILDCARD_STAR, ?KHEPRI_WILDCARD_STAR]) of diff --git a/virtuerl/src/virtuerl_mgt.erl b/virtuerl/src/virtuerl_mgt.erl index c4447b8..8aac529 100644 --- a/virtuerl/src/virtuerl_mgt.erl +++ b/virtuerl/src/virtuerl_mgt.erl @@ -11,9 +11,11 @@ -export([start_link/0]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3, handle_continue/2]). --export([create_vm/0, domain_create/1, domain_get/1, domain_delete/1]). +-export([create_vm/0, domain_create/1, domain_get/1, domain_delete/1, domain_stop/1, domain_start/1]). +-export([home_path/0]). -define(SERVER, ?MODULE). +-define(APPLICATION, virtuerl). create_vm() -> gen_server:call(?SERVER, {domain_create, {default}}). @@ -31,35 +33,63 @@ domain_get(Conf) -> domains_list(Conf) -> gen_server:call(?SERVER, {domains_list, Conf}). +domain_stop(Id) -> + gen_server:call(?SERVER, {domain_update, #{id => Id, state => stopped}}). + +domain_start(Id) -> + gen_server:call(?SERVER, {domain_update, #{id => Id, state => running}}). %%%=================================================================== %%% Spawning and gen_server implementation %%%=================================================================== --record(domain, {id, network_id, network_addrs, mac_addr, ipv4_addr, ipv6_addr, tap_name}). +home_path() -> + application:get_env(?APPLICATION, home, "var"). start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). init([]) -> - {ok, Table} = dets:open_file(vms, []), + {ok, Table} = dets:open_file(domains, [{file, filename:join(home_path(), "domains.dets")}]), %% virtuerl_ipam:ipam_put_net({default, <<192:8, 168:8, 10:8, 0:8>>, 28}), %% application:ensure_all_started(grpcbox), - {ok, {Table}, {continue, sync_domains}}. + {ok, {Table}, {continue, setup_base}}. %% {ok, {Table}}. -handle_continue(sync_domains, State) -> - {Table} = State, - TargetDomains = sets:from_list([Id || {Id, _} <- dets:match_object(Table, '_')]), +handle_continue(setup_base, State) -> + ok = filelib:ensure_path(filename:join(home_path(), "domains")), + + BaseImagePath = filename:join(home_path(), "debian-12-genericcloud-amd64-20230910-1499.qcow2"), + case filelib:is_regular(BaseImagePath) of + true -> ok; + false -> + TempImagePath = "/tmp/virtuerl/debian-12-genericcloud-amd64-20230910-1499.qcow2", + ok = filelib:ensure_dir(TempImagePath), + httpc:request(get, "https://cloud.debian.org/images/cloud/bookworm/20230910-1499/debian-12-genericcloud-arm64-20230910-1499.qcow2", [], + [{stream, TempImagePath}]), + file:rename(TempImagePath, BaseImagePath) + end, + + {noreply, State, {continue, sync_domains}}; + + +handle_continue(sync_domains, {Table} = State) -> + TargetDomains = sets:from_list([Id || {Id, Domain} <- dets:match_object(Table, '_'), + case Domain of + #{state := stopped} -> false; + _ -> true + end]), RunningDomains = sets:from_list([Id || {Id, _, _, _} <- supervisor:which_children(virtuerl_sup), is_binary(Id)]), ToDelete = sets:subtract(RunningDomains, TargetDomains), ToAdd = sets:subtract(TargetDomains, RunningDomains), + [supervisor:terminate_child(virtuerl_sup, Id) || Id <- sets:to_list(ToDelete)], [supervisor:delete_child(virtuerl_sup, Id) || Id <- sets:to_list(ToDelete)], + ok = gen_server:call(virtuerl_net, {net_update}), [ supervisor:start_child(virtuerl_sup, { Id, {virtuerl_qemu, start_link, [Id]}, - permanent, + transient, infinity, worker, [] @@ -80,7 +110,7 @@ handle_call({domain_create, Conf}, _From, State) -> {Table} = State, #{network_id := NetworkID} = Conf, DomainID = virtuerl_util:uuid4(), - Domain = #domain{id = DomainID, network_id = NetworkID}, % TODO: save ipv4 addr as well + Domain = #{id => DomainID, network_id => NetworkID}, % TODO: save ipv4 addr as well dets:insert_new(Table, {DomainID, Domain}), dets:sync(Table), @@ -104,29 +134,31 @@ handle_call({domain_create, Conf}, _From, State) -> case Addr of undefined -> Tag = KeyToTag(Key), - {ok, _, Ip} = virtuerl_ipam:assign_next(NetworkID, Tag, DomainID), - {Key, Ip}; + {ok, {NetAddr, Prefixlen}, Ip} = virtuerl_ipam:assign_next(NetworkID, Tag, DomainID), + {Key, NetAddr, Ip, Prefixlen}; Addr -> Addr1 = virtuerl_net:parse_ip(Addr), - {ok, _} = virtuerl_ipam:ipam_put_ip(NetworkID, Addr1, DomainID), - {Key, Addr1} + {ok, {NetAddr, Prefixlen}} = virtuerl_ipam:ipam_put_ip(NetworkID, Addr1, DomainID), + {Key, NetAddr, Addr1, Prefixlen} end || {Key, Addr} <- Conf2 ], - AddressesMap = maps:from_list(Addresses), + IpCidrs = [{Ip, Prefixlen} || {_, _, Ip, Prefixlen} <- Addresses], + AddressesMap = maps:from_list([{K, A} || {K, _, A, _} <- Addresses]), Ipv4Addr = maps:get(ipv4_addr, AddressesMap, undefined), Ipv6Addr = maps:get(ipv6_addr, AddressesMap, undefined), Domains = dets:match_object(Table, '_'), - TapNames = sets:from_list([Tap || #domain{tap_name=Tap} <- Domains]), + TapNames = sets:from_list([Tap || #{tap_name := Tap} <- Domains]), TapName = generate_unique_tap_name(TapNames), <> = <<(rand:uniform(16#ffffffffffff)):48>>, MacAddr = <>, - dets:insert(Table, {DomainID, Domain#domain{network_addrs =Cidrs, mac_addr=MacAddr, ipv4_addr=Ipv4Addr, ipv6_addr = Ipv6Addr, tap_name = TapName}}), + dets:insert(Table, {DomainID, Domain#{network_addrs => Cidrs, mac_addr=>MacAddr, ipv4_addr=>Ipv4Addr, ipv6_addr => Ipv6Addr, cidrs => IpCidrs, tap_name => TapName}}), dets:sync(Table), ok = gen_server:call(virtuerl_net, {net_update}), + supervisor:start_child(virtuerl_sup, { DomainID, {virtuerl_qemu, start_link, [DomainID]}, @@ -136,10 +168,19 @@ handle_call({domain_create, Conf}, _From, State) -> [] }), {reply, {ok, maps:merge(#{id => DomainID, tap_name => iolist_to_binary(TapName), mac_addr => binary:encode_hex(MacAddr)}, maps:map(fun(_, V) -> iolist_to_binary(virtuerl_net:format_ip(V)) end, AddressesMap))}, State}; +handle_call({domain_update, #{id := DomainID, state := RunState}}, _From, {Table} = State) -> + Reply = case dets:lookup(Table, DomainID) of + [{_, Domain}] -> + ok = dets:insert(Table, {DomainID, Domain#{state := RunState}}), + ok = dets:sync(Table), + ok; + [] -> notfound + end, + {reply, Reply, State, {continue, sync_domains}}; handle_call({domain_get, #{id := DomainID}}, _From, State) -> {Table} = State, Reply = case dets:lookup(Table, DomainID) of - [{_, #domain{network_id = NetworkID, mac_addr = MacAddr, ipv4_addr=IP, tap_name = TapName}}] -> + [{_, #{network_id := NetworkID, mac_addr := MacAddr, ipv4_addr:=IP, tap_name := TapName}}] -> DomRet = #{network_id => NetworkID, mac_addr => binary:encode_hex(MacAddr), ipv4_addr => virtuerl_net:format_ip_bitstring(IP), tap_name => iolist_to_binary(TapName)}, {ok, DomRet}; [] -> notfound diff --git a/virtuerl/src/virtuerl_net.erl b/virtuerl/src/virtuerl_net.erl index 1da580a..3a60067 100644 --- a/virtuerl/src/virtuerl_net.erl +++ b/virtuerl/src/virtuerl_net.erl @@ -13,7 +13,7 @@ -export([start_link/0]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --export([parse_cidr/1, format_ip/1, format_ip_bitstring/1, parse_ip/1]). +-export([parse_cidr/1, format_ip/1, format_ip_bitstring/1, parse_ip/1, bridge_addr/1, bridge_addr/2]). -define(SERVER, ?MODULE). @@ -26,7 +26,7 @@ start_link() -> init([]) -> ?LOG_INFO(#{what => "Started", who => virtuerl_net}), - {ok, Table} = dets:open_file(vms, []), + {ok, Table} = dets:open_file(domains, [{file, filename:join(virtuerl_mgt:home_path(), "domains.dets")}]), update_net(Table), % TODO: erlexec: spawn bird -f {ok, {Table}}. @@ -73,8 +73,6 @@ update_net(Table) -> [io:format("~p~n", [Domain]) || Domain <- Domains], reload_net(Table). --record(domain, {id, network_id, network_addrs, mac_addr, ipv4_addr, ipv6_addr, tap_name}). - handle_interface(If, Table) -> %% 1. Delete all devices without an address set Addrs = maps:get(<<"addr_info">>, If, []), @@ -102,7 +100,7 @@ reload_net(Table) -> io:format("Actual: ~p~n", [Matched]), Domains = dets:match_object(Table, '_'), - TargetAddrs = sets:from_list([lists:sort(network_cidrs_to_bride_cidrs(Cidrs)) || {_, #domain{network_addrs = Cidrs}} <- Domains]), + TargetAddrs = sets:from_list([lists:sort(network_cidrs_to_bride_cidrs(Cidrs)) || {_, #{network_addrs := Cidrs}} <- Domains]), io:format("Target: ~p~n", [sets:to_list(TargetAddrs)]), update_nftables(Domains), sync_networks(Matched, TargetAddrs), @@ -112,7 +110,7 @@ reload_net(Table) -> ok. update_nftables(Domains) -> - BridgeAddrs = lists:flatten([network_cidrs_to_bride_addrs(Cidrs) || {_, #domain{network_addrs = Cidrs}} <- Domains]), + BridgeAddrs = lists:uniq(lists:flatten([network_cidrs_to_bride_addrs(Cidrs) || {_, #{network_addrs := Cidrs}} <- Domains])), BridgeAddrsTyped = lists:map(fun (Addr) -> case binary:match(Addr, <<":">>) of nomatch -> {ipv4, Addr}; @@ -168,6 +166,9 @@ bridge_addr(<>) -> BitSize = bit_size(Addr), <> = Addr, <<(AddrInt+1):BitSize>>. +bridge_addr(<>, Prefixlen) -> + <> = Addr, + <>. parse_cidr(<>) -> parse_cidr(binary_to_list(CIDR)); parse_cidr(CIDR) -> @@ -202,7 +203,7 @@ update_bird_conf(Domains) -> Output = os:cmd("ip -j addr"), {ok, JSON} = thoas:decode(Output), Bridges = maps:from_list([get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), <<"verlbr">>)]), - AddrMap = maps:from_list([{Addr, {bridge_addr(NetAddr), Prefixlen}} || {_, #domain{network_addrs = {NetAddr, Prefixlen}, ipv4_addr = Addr}} <- Domains]), + AddrMap = maps:from_list([{Addr, {bridge_addr(NetAddr), Prefixlen}} || {_, #{network_addrs := {NetAddr, Prefixlen}, ipv4_addr := Addr}} <- Domains]), AddrToBridgeMap = maps:map(fun (_, Net) -> maps:get(Net, Bridges) end, AddrMap), io:format("DOMAINS: ~p~n", [Domains]), @@ -258,9 +259,10 @@ sync_taps(Domains) -> {ok, JSONTaps} = thoas:decode(OutputTaps), io:format("TAPS: ~p~n", [JSONTaps]), TapsActual = sets:from_list([maps:get(<<"ifname">>, L) || L <- JSONTaps, startswith(maps:get(<<"ifname">>, L), <<"verltap">>)]), - TapsTarget = sets:from_list([TapName || {_, #domain{tap_name = TapName}} <- Domains]), - io:format("Taps to add: ~p~n", [sets:to_list(TapsTarget)]), - TapsMap = maps:from_list([{Tap, {MacAddr, network_cidrs_to_bride_cidrs(Cidrs)}} || {_, #domain{network_addrs = Cidrs, tap_name = Tap, mac_addr = MacAddr}} <- Domains]), + TapsTarget = sets:from_list([iolist_to_binary(TapName) || {_, #{tap_name := TapName}} <- Domains]), % TODO: persist tap_name as binary + io:format("Taps Target: ~p~n", [sets:to_list(TapsTarget)]), + io:format("Taps Actual: ~p~n", [sets:to_list(TapsActual)]), + TapsMap = maps:from_list([{iolist_to_binary(Tap), {MacAddr, network_cidrs_to_bride_cidrs(Cidrs)}} || {_, #{network_addrs := Cidrs, tap_name := Tap, mac_addr := MacAddr}} <- Domains]), io:format("TapsMap: ~p~n", [TapsMap]), TapsToDelete = sets:subtract(TapsActual, TapsTarget), @@ -281,8 +283,7 @@ end, sets:to_list(TapsToDelete)), add_taps(M) when is_map(M) -> add_taps(maps:to_list(M)); add_taps([]) -> ok; add_taps([{Tap, {Mac, Bridge}}|T]) -> - <> = binary:encode_hex(Mac), - MacAddrString = <>, + MacAddrString = virtuerl_util:mac_to_str(Mac), Cmd = io_lib:format("ip tuntap add dev ~s mode tap~nip link set dev ~s address ~s master ~s~nip link set ~s up~n", [Tap, Tap, MacAddrString, Bridge, Tap]), io:format(Cmd), os:cmd(Cmd), diff --git a/virtuerl/src/virtuerl_qemu.erl b/virtuerl/src/virtuerl_qemu.erl index f7194e4..54def68 100644 --- a/virtuerl/src/virtuerl_qemu.erl +++ b/virtuerl/src/virtuerl_qemu.erl @@ -8,9 +8,10 @@ -behaviour(gen_server). --export([start_link/1]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). +-export([start_link/1, callback_mode/0]). +-export([init/1, terminate/2]). +-export([handle_continue/2, handle_info/2]). +-export([handle_call/3, handle_cast/2]). -define(SERVER, ?MODULE). @@ -20,51 +21,179 @@ start_link(ID) -> %% gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - Pid = spawn_link(fun() -> - io:format("QEMU: Starting VM with ID ~p~n", [ID]), - timer:sleep(20000), - io:format("QEMU: Exiting ~p~n", [ID]), - timer:sleep(500), - exit(failure) - - end), - {ok, Pid}. - -init([]) -> - {ok, Table} = dets:open_file(vms, []), - % TODO: erlexec: spawn bird -f - {ok, {Table}}. - -terminate(_Reason, {Table}) -> - dets:close(Table). +%% Pid = spawn_link(fun() -> +%% io:format("QEMU: Starting VM with ID ~p~n", [ID]), +%% timer:sleep(20000), +%% io:format("QEMU: Exiting ~p~n", [ID]), +%% timer:sleep(500), +%% exit(failure) +%% +%% end), +%% {ok, Pid}. + + gen_server:start_link(?MODULE, [ID], []). + +callback_mode() -> + handle_event_function. + +init([ID]) -> + {ok, Table} = dets:open_file(domains, [{file, filename:join(virtuerl_mgt:home_path(), "domains.dets")}]), + [{DomainId, #{mac_addr:=MacAddr, tap_name := TapName} = Domain}] = dets:lookup(Table, ID), + DomainHomePath = filename:join([virtuerl_mgt:home_path(), "domains", DomainId]), + ok = filelib:ensure_path(DomainHomePath), + RootVolumePath = filename:join(DomainHomePath, "root.qcow2"), + case filelib:is_regular(RootVolumePath) of + false -> + BaseImagePath = filename:join([virtuerl_mgt:home_path(), "debian-12-genericcloud-amd64-20230910-1499.qcow2"]), + exec:run(io_lib:format("qemu-img create -f qcow2 -b ~s -F qcow2 ~s", [filename:absname(BaseImagePath), RootVolumePath])); + _ -> noop + end, + process_flag(trap_exit, true), + ensure_cloud_config(Domain), + file:delete(filename:join(DomainHomePath, "qmp.sock")), + file:delete(filename:join(DomainHomePath, "serial.sock")), + Cmd = iolist_to_binary(["kvm -no-shutdown -S -nic tap,ifname=",TapName,",script=no,downscript=no,model=virtio-net-pci,mac=",virtuerl_util:mac_to_str(MacAddr), " -display none -m 512 -drive file=root.qcow2,if=virtio -drive driver=raw,file=cloud_config.iso,if=virtio -display none -qmp unix:qmp.sock,server=on,wait=off -serial unix:serial.sock,server=on,wait=off"]), + io:format("QEMU cmdline: ~s~n", [Cmd]), + {ok, Pid, OsPid} = exec:run_link(Cmd, [{cd, DomainHomePath}]), + State = #{table => Table, id => ID, domain => Domain, qemu_pid => {Pid, OsPid}, qmp_pid => undefined}, + {ok, State, {continue, setup_qmp}}. + +handle_continue(setup_qmp, #{id := ID} = State) -> + QmpSocketPath = filename:join([virtuerl_mgt:home_path(), "domains", ID, "qmp.sock"]), + io:format("waiting for qmp.sock ~p~n", [erlang:timestamp()]), +%% {ok, _} = exec:run(iolist_to_binary(["inotifywait -e create --include 'qmp\\.sock' ", ID]), [sync]), + ok = wait_for_socket(ID), + io:format("done waiting for qmp.sock ~p~n", [erlang:timestamp()]), + {ok, QmpPid} = virtuerl_qmp:start_link(QmpSocketPath, self()), + virtuerl_qmp:exec(QmpPid, cont), + {noreply, State#{qmp_pid => QmpPid}}. -handle_call({vm_create, Conf}, _From, State) -> - {reply, ok, State}. +handle_info({qmp, Event}, #{table := Table, id := ID, domain := Domain} = State) -> + io:format("QMP: ~p~n", [Event]), + case Event of + #{<<"event">> := <<"STOP">>} -> + [{DomainId, Domain}] = dets:lookup(Table, ID), + DomainUpdated = Domain#{state => stopped}, + ok = dets:insert(Table, {DomainId, DomainUpdated}), + ok = dets:sync(Table), + {stop, normal, State#{domain => DomainUpdated}}; + _ -> {noreply, State} + end. -handle_cast({net_update}, State) -> - {Table} = State, - update_bird_conf(Table), - {noreply, State}. +shutdown_events() -> + receive + {qmp, Event} -> + io:format("shutdown QMP: ~p~n", [Event]), + shutdown_events() + after 5000 -> ok + end. -handle_info(_Info, State) -> - {noreply, State}. +exit_events() -> + receive + {'EXIT', _Pid, _Reason} -> + io:format("EXIT ~p ~p!~n", [_Pid, _Reason]), + exit_events() + after 10000 -> ok + end. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +terminate(_Reason, #{table := Table, id := ID, domain := Domain, qemu_pid := {Pid, OsPid}, qmp_pid := QmpPid}) -> + io:format("GRACEFUL SHUTDOWN: ~s (~p)~n", [ID, _Reason]), + case _Reason of + normal -> ok; + _ -> % supervisor sends "shutdown" + virtuerl_qmp:exec(QmpPid, system_powerdown), +%% shutdown_events(), + receive + {qmp, #{<<"event">> := <<"STOP">>}} -> ok + after 5000 -> timeout + end + end, +%% {ok, #{<<"return">> := #{}}} = thoas:decode(PowerdownRes), + ok = virtuerl_qmp:stop(QmpPid), + ok = exec:stop(Pid), + receive + {'EXIT', Pid, _} -> + io:format("QEMU OS process stopped!~n"), + ok; + {'EXIT', OsPid, _} -> + io:format("QEMU OS process stopped (OsPid)!~n"), + ok + after 5000 -> timeout + end, +%% exit_events(), + dets:close(Table). %%%=================================================================== %%% Internal functions %%%=================================================================== -update_bird_conf(Table) -> - % 1. write config - %% for VM in VMs: - %% append VM.IP to static routes: VM.IP via $VM.network.bridge - % 2. birdc configure - % 3. profit? - VMs = dets:match_object(Table, '_'), - [io:format("~p~n", [VM]) || VM <- VMs], - reload_bird(). - -reload_bird() -> +wait_for_socket(ID) -> + QmpSocketPath = filename:join([virtuerl_mgt:home_path(), "domains", ID, "qmp.sock"]), + io:format("checking...~n"), + case filelib:last_modified(QmpSocketPath) of + 0 -> + timer:sleep(20), + wait_for_socket(ID); + _ -> ok + end. + +ensure_cloud_config(#{id := DomainID} = Domain) -> + case filelib:is_regular(filename:join([virtuerl_mgt:home_path(), "domains", DomainID, "cloud_config.iso"])) of + true -> ok; + false -> create_cloud_config(Domain) + end. + +create_cloud_config(#{id := DomainID, mac_addr := MacAddr, cidrs := Cidrs}) -> + NetConf = [ + "version: 2\n", + "ethernets:\n", + " primary:\n", + " match:\n", + " macaddress: \"", virtuerl_util:mac_to_str(MacAddr), "\"\n", + " set-name: ens2\n", + " dhcp4: false\n", + " dhcp6: false\n", + " addresses:\n", [[ + " - ", virtuerl_net:format_ip(IpAddr), "/", integer_to_binary(Prefixlen), "\n"] || {IpAddr, Prefixlen} <- Cidrs], + " routes:\n", [[ + " - to: default\n", + " via: ", virtuerl_net:format_ip(virtuerl_net:bridge_addr(IpAddr, Prefixlen)), "\n"] || {IpAddr, Prefixlen} <- Cidrs], + "" + ], + MetaData = [ + "instance-id: ", DomainID, "\n", + "local-hostname: ", DomainID, "\n" + ], + % openssl passwd -6 + UserData = [ + "#cloud-config\n", + "users:\n", + " - default\n", + " - name: ilya\n", + " passwd: $6$VAKAlO91OquUA1ON$4w2.omQG9OUt0KuMtCvYrJgervyK8WZrTMFUhJI2fXTsfMN5YvuxKaZbSJTXkJvq4qAiFHptt6zKg2hTMOrkH0\n", + " lock_passwd: false\n", + " shell: /bin/bash\n", + " sudo: ALL=(ALL) NOPASSWD:ALL\n", + " ssh_authorized_keys:\n", + " - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDCKxHNpVg1whPegPv0KcRQTOfyVIqLwvMfVLyT9OpBPXHDudsFz9soOgMUEyWm8ZJ+pJ9fRCg66B+D5/ZRTwJCBpyNncfXCwu8xEJgEeoIubObh6t6dHWqqxX/yhHAS5GIRUSypm78qg6V+SQ6SeJXSjOCLAbZmhyWgJrlDm9M6GTPQhPAztrgsCUrzxIpZ5el5BwJXrm3I+LOmofAUqgbLQz9HuGJzPpnfABDa9WoVfI0L7oTr0qGpWwx8l71b2s8AYl7GMD/bEkZKyi9SSwEVCHA88F7dYYrZ3+fMXE/mJf+v0ece2lIDT7Te1gtqiLu/izJNmqD+b6mtnnXxVxNOtynhv3t6uLE9kBX22SBCCRqPJzETGNXvYH6fATEe88dhLh8kTppLRB5UGUd/zztxuNBSpMwFXaq8SlTKURxvF8BuFIPCz0FW8fq+TA/xZfBYsiVt59jXgl6BQyEGY4bMuMtT2nD8QXwZ5vsj52mzKGJwBwduiaX302brHYUyQkuyLII5iqmCNZ5YLlMY76a61Yg9pWMeRwQscSO2k4a18GOo+sIrQVTyUQiT3KhRRaDNrZuCPicQRgkJuiS1fKt1cWjnOlyweLxSYbpKnoS0H7vt+NrtbU1u9FPknXQPQ0pxixPpV3zgUdfOLmisFH7WGVjwNVvZAlNc5uyqm0fbw== ilya@verbit.io\n", + "" + ], + DomainBasePath = filename:join([virtuerl_mgt:home_path(), "domains", DomainID]), + IsoBasePath = filename:join(DomainBasePath, "iso"), + ok = filelib:ensure_path(IsoBasePath), + NetConfPath = filename:join(IsoBasePath, "network-config"), + ok = file:write_file(NetConfPath, NetConf), + MetaDataPath = filename:join(IsoBasePath, "meta-data"), + ok = file:write_file(MetaDataPath, MetaData), + UserDataPath = filename:join(IsoBasePath, "user-data"), + ok = file:write_file(UserDataPath, UserData), + IsoCmd = ["genisoimage -output ", filename:join(DomainBasePath, "cloud_config.iso"), " -volid cidata -joliet -rock ", UserDataPath, " ", MetaDataPath, " ", NetConfPath], + os:cmd(binary_to_list(iolist_to_binary(IsoCmd))), + ok = file:del_dir_r(IsoBasePath), ok. + +handle_call(Request, From, State) -> + erlang:error(not_implemented). + +handle_cast(Request, State) -> + erlang:error(not_implemented). diff --git a/virtuerl/src/virtuerl_qmp.erl b/virtuerl/src/virtuerl_qmp.erl new file mode 100644 index 0000000..c1c736b --- /dev/null +++ b/virtuerl/src/virtuerl_qmp.erl @@ -0,0 +1,97 @@ +%%%------------------------------------------------------------------- +%%% @author ilya +%%% @copyright (C) 2023, +%%% @doc +%%% @end +%%%------------------------------------------------------------------- +-module(virtuerl_qmp). + +-behaviour(gen_server). + +-export([start_link/2, handle_cast/2]). +-export([init/1, terminate/2]). +-export([handle_call/3, handle_info/2]). +-export([exec/2, stop/1]). + +-define(SERVER, ?MODULE). + +exec(Pid, Command) -> + gen_server:call(Pid, Command). + +stop(Pid) -> + gen_server:stop(Pid). + +%%%=================================================================== +%%% Spawning and gen_server implementation +%%%=================================================================== + +start_link(QmpSocketPath, Receiver) -> + gen_server:start_link(?MODULE, {QmpSocketPath, Receiver}, []). + +init({QmpSocketPath, Receiver}) -> + Self = self(), + Pid = spawn_link(fun () -> qmp_translator(QmpSocketPath, Self) end), + io:format("virtuerl_qmp: init~n"), + receive + {qmp, #{<<"QMP">> := #{}}} -> ok + after 1000 -> exit(qmp_not_responding) + end, + io:format("virtuerl_qmp: init after~n"), + execute(Pid, qmp_capabilities), + io:format("virtuerl_qmp: qmp caps~n"), + {ok, {Pid, Receiver}}. + +terminate(_Reason, {Pid, _}) -> + Ref = monitor(process, Pid), + exit(Pid, normal), + receive + {'DOWN', Ref, process, Pid, normal} -> ok + end, + true = demonitor(Ref), + io:format("exiting QMP server~n"). + +handle_call(Command, _From, {Pid, _} = State) -> + execute(Pid, Command), + {reply, ok, State}. + +handle_cast(Request, State) -> + erlang:error(not_implemented). + +handle_info({qmp, #{<<"event">> := _} = Event}, {_, Receiver} = State) -> + Receiver ! {qmp, Event}, + {noreply, State}. + +execute(Pid, Command) when is_pid(Pid) -> + Pid ! {qmp, Command}, + receive + {qmp, #{<<"return">> := #{}}} -> ok + end. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +qmp_translator(QmpSocketPath, Receiver) -> + process_flag(trap_exit, true), + io:format("starting QMP translator (~s)!~n", [QmpSocketPath]), + {ok, QmpSocket} = gen_tcp:connect({local, QmpSocketPath}, 0, [local, {active, true}]), + qmp_loop(QmpSocket, Receiver). + +qmp_loop(QmpSocket, Receiver) -> + receive + {tcp, _Socket, RawData} -> + io:format("qmp_loop/tcp/raw: ~p~n", [RawData]), + Lines = re:split(RawData, "\r?\n", [trim]), + Jsons = lists:map(fun (Line) -> {ok, Json} = thoas:decode(Line), Json end, Lines), + io:format("qmp_loop/tcp: ~p~n", [Jsons]), + [Receiver ! {qmp, Json} || Json <- Jsons], + qmp_loop(QmpSocket, Receiver); + {qmp, Command} when is_atom(Command) -> + io:format("qmp_loop/qmp: ~p~n", [Command]), + ok = gen_tcp:send(QmpSocket, thoas:encode(#{execute => Command})), + qmp_loop(QmpSocket, Receiver); + {'EXIT', _SenderID, Reason} -> + io:format("closing QMP socket (~p)!~n", [Reason]), + ok = gen_tcp:close(QmpSocket), + exit(Reason) + end. diff --git a/virtuerl/src/virtuerl_stor.erl b/virtuerl/src/virtuerl_stor.erl new file mode 100644 index 0000000..3862075 --- /dev/null +++ b/virtuerl/src/virtuerl_stor.erl @@ -0,0 +1,48 @@ +%%%------------------------------------------------------------------- +%%% @author ilya +%%% @copyright (C) 2023, +%%% @doc +%%% @end +%%%------------------------------------------------------------------- +-module(virtuerl_stor). + +-behaviour(gen_server). + +-export([start_link/0]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +%%%=================================================================== +%%% Spawning and gen_server implementation +%%%=================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +init([]) -> + {ok, Table} = dets:open_file(stors, []), + {ok, {Table}}. + +handle_call({domain_delete, #{id := DomainID}}, _From, State) -> + {Table} = State, + Res = dets:delete(Table, DomainID), + dets:sync(Table), + ok = supervisor:terminate_child(virtuerl_sup, DomainID), + ok = supervisor:delete_child(virtuerl_sup, DomainID), + ok = gen_server:call(virtuerl_net, {net_update}), + {reply, Res, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, {Table}) -> + dets:close(Table), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/virtuerl/src/virtuerl_util.erl b/virtuerl/src/virtuerl_util.erl index b06cad2..0880de6 100644 --- a/virtuerl/src/virtuerl_util.erl +++ b/virtuerl/src/virtuerl_util.erl @@ -10,10 +10,21 @@ -author("ilya-stroeer"). %% API --export([uuid4/0]). +-export([uuid4/0, mac_to_str/1, delete_file/1]). uuid4() -> ID = string:lowercase(binary:encode_hex(<<(rand:uniform(16#FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)-1):128>>)), <> = ID, Uuid4 = iolist_to_binary([A, "-", B, "-", C, "-", D, "-", E]), Uuid4. + +mac_to_str(<>) -> + <> = string:lowercase(binary:encode_hex(<>)), + <>. + +delete_file(Filename) -> + case file:delete(Filename) of + ok -> ok; + {error, enoent} -> ok; + Other -> Other + end.