virtuerl: wip

.github/workflows/build.yaml

@@ -39,16 +39,16 @@ jobs:
       - name: Setup Virtuerl
         working-directory: virtuerl
         run: |
-          sudo -s rebar3 compile
+          rebar3 release
 
           cat << EOF | sudo tee -a /etc/systemd/system/virtuerl.service
           [Unit]
           Description=Virtuerl
           After=network.target
 
           [Service]
-          WorkingDirectory=${PWD}
-          ExecStart=/bin/sh -c 'erl -pa _build/default/lib/*/ebin -config config/sys.config -s virtuerl_app -noshell -noinput'
+          WorkingDirectory=${PWD}/_build/default/rel/virtuerl/
+          ExecStart=bin/virtuerl foreground
           Restart=always
 
           [Install]

virtuerl/config/sys.config

@@ -2,7 +2,7 @@
   [{logger_level, all},
     {logger,
       [{handler, default, logger_std_h,
-        #{ level => info,
+        #{ level => debug,
             formatter => {logger_formatter, #{single_line => false}}}}
       ]}]},
   {grpcbox, [
@@ -11,5 +11,6 @@
       {keyfile, "config/client.key"},
       {cacertfile, "config/ca.crt"}
     ]}], #{}}]}}
-  ]}
+  ]},
+  {erlexec, [{root, true}, {user, "root"}]}
 ].

virtuerl/rebar.config

@@ -4,7 +4,13 @@
     {thoas, "1.0.0"},
     {grpcbox, "0.16.0"},
     {cowboy, "2.10.0"},
-    {mochiweb, "3.1.2"}
+    {mochiweb, "3.1.2"},
+    {erlexec, "~> 2.0"}
+]}.
+
+{relx, [
+    {release, {virtuerl, git}, [virtuerl, {khepri, load}, {mnesia, load}]},
+    {mode, prod}
 ]}.
 
 {plugins, [grpcbox_plugin]}.

virtuerl/rebar.lock

@@ -5,6 +5,7 @@
  {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.10.0">>},0},
  {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.12.1">>},1},
  {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1},
+ {<<"erlexec">>,{pkg,<<"erlexec">>,<<"2.0.2">>},0},
  {<<"gen_batch_server">>,{pkg,<<"gen_batch_server">>,<<"0.8.8">>},2},
  {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
  {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},0},
@@ -24,6 +25,7 @@
  {<<"cowboy">>, <<"FF9FFEFF91DAE4AE270DD975642997AFE2A1179D94B1887863E43F681A203E26">>},
  {<<"cowlib">>, <<"A9FA9A625F1D2025FE6B462CB865881329B5CAFF8F1854D1CBC9F9533F00E1E1">>},
  {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>},
+ {<<"erlexec">>, <<"995E40477DE94C37EC1264CC3E52EB6273938E80C9BCC4F94110A3F1C0D9ABA3">>},
  {<<"gen_batch_server">>, <<"7840A1FA63EE1EFFC83E8A91D22664847A2BA1192D30EAFFFD914ACB51578068">>},
  {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
  {<<"grpcbox">>, <<"B83F37C62D6EECA347B77F9B1EC7E9F62231690CDFEB3A31BE07CD4002BA9C82">>},
@@ -42,6 +44,7 @@
  {<<"cowboy">>, <<"3AFDCCB7183CC6F143CB14D3CF51FA00E53DB9EC80CDCD525482F5E99BC41D6B">>},
  {<<"cowlib">>, <<"163B73F6367A7341B33C794C4E88E7DBFE6498AC42DCD69EF44C5BC5507C8DB0">>},
  {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>},
+ {<<"erlexec">>, <<"CC829A7C6C23D399832DA2E998EA5EBC552232A6FE3EB1EDB400178EC8287DCB">>},
  {<<"gen_batch_server">>, <<"C3E6A1A2A0FB62AEE631A98CFA0FD8903E9562422CBF72043953E2FB1D203017">>},
  {<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>},
  {<<"grpcbox">>, <<"294DF743AE20A7E030889F00644001370A4F7CE0121F3BBDAF13CF3169C62913">>},

virtuerl/src/virtuerl.app.src

@@ -6,7 +6,11 @@
   {applications,
    [kernel,
     stdlib,
-    cowboy
+    inets,
+    mochiweb,
+    cowboy,
+    thoas,
+    erlexec
    ]},
   {env,[]},
   {modules, []},

virtuerl/src/virtuerl_app.erl

@@ -19,10 +19,12 @@ start(_StartType, _StartArgs) ->
 %%    Res = cowboy:start_clear(my_listener, [{port, 8080}], #{env => #{dispatch => Dispatch}}),
 %%    io:format("RESULT: ~p~n", [Res]),
 %%    {ok, _} = Res,
+%%    exec:debug(4),
     virtuerl_sup:start_link().
 
 start() ->
     application:ensure_all_started(virtuerl).
+%%    exec:debug(4).
 
 stop(_State) ->
     ok.

virtuerl/src/virtuerl_ipam.erl

@@ -96,19 +96,13 @@ start_link() ->
 
 init([]) ->
 	io:format("starting IPAM service~n"),
-	{ok, StoreId} = khepri:start(),
+	{ok, StoreId} = khepri:start(filename:join(virtuerl_mgt:home_path(), "khepri")),
 	init([StoreId]);
 init([StoreId]) ->
 	{ok, StoreId}.
 
 terminate(_Reason, StoreId) ->
-	DefaultStoreId = khepri_cluster:get_default_store_id(),
-	case StoreId of
-		DefaultStoreId ->
-			khepri:stop(StoreId);
-		_ ->
-		ok
-	end.
+	khepri:stop(StoreId).
 
 handle_call(net_list, _From, StoreId) ->
 	case khepri:get_many(StoreId, [network, ?KHEPRI_WILDCARD_STAR, ?KHEPRI_WILDCARD_STAR]) of

virtuerl/src/virtuerl_mgt.erl

@@ -11,9 +11,11 @@
 -export([start_link/0]).
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
   code_change/3, handle_continue/2]).
--export([create_vm/0, domain_create/1, domain_get/1, domain_delete/1]).
+-export([create_vm/0, domain_create/1, domain_get/1, domain_delete/1, domain_stop/1, domain_start/1]).
+-export([home_path/0]).
 
 -define(SERVER, ?MODULE).
+-define(APPLICATION, virtuerl).
 
 create_vm() ->
   gen_server:call(?SERVER, {domain_create, {default}}).
@@ -31,35 +33,63 @@ domain_get(Conf) ->
 domains_list(Conf) ->
   gen_server:call(?SERVER, {domains_list, Conf}).
 
+domain_stop(Id) ->
+  gen_server:call(?SERVER, {domain_update, #{id => Id, state => stopped}}).
+
+domain_start(Id) ->
+  gen_server:call(?SERVER, {domain_update, #{id => Id, state => running}}).
 
 
 %%%===================================================================
 %%% Spawning and gen_server implementation
 %%%===================================================================
 
--record(domain, {id, network_id, network_addrs, mac_addr, ipv4_addr, ipv6_addr, tap_name}).
+home_path() ->
+  application:get_env(?APPLICATION, home, "var").
 
 start_link() ->
   gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
 
 init([]) ->
-  {ok, Table} = dets:open_file(vms, []),
+  {ok, Table} = dets:open_file(domains, [{file, filename:join(home_path(), "domains.dets")}]),
 %%  virtuerl_ipam:ipam_put_net({default, <<192:8, 168:8, 10:8, 0:8>>, 28}),
 %%  application:ensure_all_started(grpcbox),
-  {ok, {Table}, {continue, sync_domains}}.
+  {ok, {Table}, {continue, setup_base}}.
 %%  {ok, {Table}}.
 
-handle_continue(sync_domains, State) ->
-  {Table} = State,
-  TargetDomains = sets:from_list([Id || {Id, _} <- dets:match_object(Table, '_')]),
+handle_continue(setup_base, State) ->
+  ok = filelib:ensure_path(filename:join(home_path(), "domains")),
+
+  BaseImagePath = filename:join(home_path(), "debian-12-genericcloud-amd64-20230910-1499.qcow2"),
+  case filelib:is_regular(BaseImagePath) of
+    true -> ok;
+    false ->
+      TempImagePath = "/tmp/virtuerl/debian-12-genericcloud-amd64-20230910-1499.qcow2",
+      ok = filelib:ensure_dir(TempImagePath),
+      httpc:request(get, "https://cloud.debian.org/images/cloud/bookworm/20230910-1499/debian-12-genericcloud-arm64-20230910-1499.qcow2", [],
+        [{stream, TempImagePath}]),
+      file:rename(TempImagePath, BaseImagePath)
+  end,
+
+  {noreply, State, {continue, sync_domains}};
+
+
+handle_continue(sync_domains, {Table} = State) ->
+  TargetDomains = sets:from_list([Id || {Id, Domain} <- dets:match_object(Table, '_'),
+    case Domain of
+      #{state := stopped} -> false;
+      _ -> true
+    end]),
   RunningDomains = sets:from_list([Id || {Id, _, _, _} <- supervisor:which_children(virtuerl_sup), is_binary(Id)]),
   ToDelete = sets:subtract(RunningDomains, TargetDomains),
   ToAdd = sets:subtract(TargetDomains, RunningDomains),
+  [supervisor:terminate_child(virtuerl_sup, Id) || Id <- sets:to_list(ToDelete)],
   [supervisor:delete_child(virtuerl_sup, Id) || Id <- sets:to_list(ToDelete)],
+  ok = gen_server:call(virtuerl_net, {net_update}),
   [  supervisor:start_child(virtuerl_sup, {
     Id,
     {virtuerl_qemu, start_link, [Id]},
-    permanent,
+    transient,
     infinity,
     worker,
     []
@@ -80,7 +110,7 @@ handle_call({domain_create, Conf}, _From, State) ->
   {Table} = State,
   #{network_id := NetworkID} = Conf,
   DomainID = virtuerl_util:uuid4(),
-  Domain = #domain{id = DomainID, network_id = NetworkID},  % TODO: save ipv4 addr as well
+  Domain = #{id => DomainID, network_id => NetworkID},  % TODO: save ipv4 addr as well
   dets:insert_new(Table, {DomainID, Domain}),
   dets:sync(Table),
 
@@ -104,29 +134,31 @@ handle_call({domain_create, Conf}, _From, State) ->
     case Addr of
       undefined ->
         Tag = KeyToTag(Key),
-        {ok, _, Ip} = virtuerl_ipam:assign_next(NetworkID, Tag, DomainID),
-        {Key, Ip};
+        {ok, {NetAddr, Prefixlen}, Ip} = virtuerl_ipam:assign_next(NetworkID, Tag, DomainID),
+        {Key, NetAddr, Ip, Prefixlen};
       Addr ->
         Addr1 = virtuerl_net:parse_ip(Addr),
-        {ok, _} = virtuerl_ipam:ipam_put_ip(NetworkID, Addr1, DomainID),
-        {Key, Addr1}
+        {ok, {NetAddr, Prefixlen}} = virtuerl_ipam:ipam_put_ip(NetworkID, Addr1, DomainID),
+        {Key, NetAddr, Addr1, Prefixlen}
     end
     || {Key, Addr} <- Conf2
   ],
-  AddressesMap = maps:from_list(Addresses),
+  IpCidrs = [{Ip, Prefixlen} || {_, _, Ip, Prefixlen} <- Addresses],
+  AddressesMap = maps:from_list([{K, A} || {K, _, A, _} <- Addresses]),
   Ipv4Addr = maps:get(ipv4_addr, AddressesMap, undefined),
   Ipv6Addr = maps:get(ipv6_addr, AddressesMap, undefined),
 
   Domains = dets:match_object(Table, '_'),
-  TapNames = sets:from_list([Tap || #domain{tap_name=Tap} <- Domains]),
+  TapNames = sets:from_list([Tap || #{tap_name := Tap} <- Domains]),
   TapName = generate_unique_tap_name(TapNames),
   <<A:6, _:2, B:40>> = <<(rand:uniform(16#ffffffffffff)):48>>,
   MacAddr = <<A:6, 2:2, B:40>>,
 
-  dets:insert(Table, {DomainID, Domain#domain{network_addrs =Cidrs, mac_addr=MacAddr, ipv4_addr=Ipv4Addr, ipv6_addr = Ipv6Addr, tap_name = TapName}}),
+  dets:insert(Table, {DomainID, Domain#{network_addrs => Cidrs, mac_addr=>MacAddr, ipv4_addr=>Ipv4Addr, ipv6_addr => Ipv6Addr, cidrs => IpCidrs, tap_name => TapName}}),
   dets:sync(Table),
 
   ok = gen_server:call(virtuerl_net, {net_update}),
+
   supervisor:start_child(virtuerl_sup, {
     DomainID,
     {virtuerl_qemu, start_link, [DomainID]},
@@ -136,10 +168,19 @@ handle_call({domain_create, Conf}, _From, State) ->
     []
   }),
   {reply, {ok, maps:merge(#{id => DomainID, tap_name => iolist_to_binary(TapName), mac_addr => binary:encode_hex(MacAddr)}, maps:map(fun(_, V) -> iolist_to_binary(virtuerl_net:format_ip(V)) end, AddressesMap))}, State};
+handle_call({domain_update, #{id := DomainID, state := RunState}}, _From, {Table} = State) ->
+  Reply = case dets:lookup(Table, DomainID) of
+            [{_, Domain}] ->
+              ok = dets:insert(Table, {DomainID, Domain#{state := RunState}}),
+              ok = dets:sync(Table),
+              ok;
+            [] -> notfound
+          end,
+  {reply, Reply, State, {continue, sync_domains}};
 handle_call({domain_get, #{id := DomainID}}, _From, State) ->
   {Table} = State,
   Reply = case dets:lookup(Table, DomainID) of
-    [{_, #domain{network_id = NetworkID, mac_addr = MacAddr, ipv4_addr=IP, tap_name = TapName}}] ->
+    [{_, #{network_id := NetworkID, mac_addr := MacAddr, ipv4_addr:=IP, tap_name := TapName}}] ->
       DomRet = #{network_id => NetworkID, mac_addr => binary:encode_hex(MacAddr), ipv4_addr => virtuerl_net:format_ip_bitstring(IP), tap_name => iolist_to_binary(TapName)},
       {ok, DomRet};
     [] -> notfound

virtuerl/src/virtuerl_net.erl

@@ -13,7 +13,7 @@
 -export([start_link/0]).
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
   code_change/3]).
--export([parse_cidr/1, format_ip/1, format_ip_bitstring/1, parse_ip/1]).
+-export([parse_cidr/1, format_ip/1, format_ip_bitstring/1, parse_ip/1, bridge_addr/1, bridge_addr/2]).
 
 -define(SERVER, ?MODULE).
 
@@ -26,7 +26,7 @@ start_link() ->
 
 init([]) ->
   ?LOG_INFO(#{what => "Started", who => virtuerl_net}),
-  {ok, Table} = dets:open_file(vms, []),
+  {ok, Table} = dets:open_file(domains, [{file, filename:join(virtuerl_mgt:home_path(), "domains.dets")}]),
   update_net(Table),
   % TODO: erlexec: spawn bird -f
   {ok, {Table}}.
@@ -73,8 +73,6 @@ update_net(Table) ->
   [io:format("~p~n", [Domain]) || Domain <- Domains],
   reload_net(Table).
 
--record(domain, {id, network_id, network_addrs, mac_addr, ipv4_addr, ipv6_addr, tap_name}).
-
 handle_interface(If, Table) ->
   %% 1. Delete all devices without an address set
   Addrs = maps:get(<<"addr_info">>, If, []),
@@ -102,7 +100,7 @@ reload_net(Table) ->
   io:format("Actual: ~p~n", [Matched]),
   Domains = dets:match_object(Table, '_'),
 
-  TargetAddrs = sets:from_list([lists:sort(network_cidrs_to_bride_cidrs(Cidrs)) || {_, #domain{network_addrs = Cidrs}} <- Domains]),
+  TargetAddrs = sets:from_list([lists:sort(network_cidrs_to_bride_cidrs(Cidrs)) || {_, #{network_addrs := Cidrs}} <- Domains]),
   io:format("Target: ~p~n", [sets:to_list(TargetAddrs)]),
   update_nftables(Domains),
   sync_networks(Matched, TargetAddrs),
@@ -112,7 +110,7 @@ reload_net(Table) ->
   ok.
 
 update_nftables(Domains) ->
-  BridgeAddrs = lists:flatten([network_cidrs_to_bride_addrs(Cidrs) || {_, #domain{network_addrs = Cidrs}} <- Domains]),
+  BridgeAddrs = lists:uniq(lists:flatten([network_cidrs_to_bride_addrs(Cidrs) || {_, #{network_addrs := Cidrs}} <- Domains])),
   BridgeAddrsTyped = lists:map(fun (Addr) ->
     case binary:match(Addr, <<":">>) of
       nomatch -> {ipv4, Addr};
@@ -168,6 +166,9 @@ bridge_addr(<<Addr/binary>>) ->
   BitSize = bit_size(Addr),
   <<AddrInt:BitSize>> = Addr,
   <<(AddrInt+1):BitSize>>.
+bridge_addr(<<Addr/binary>>, Prefixlen) ->
+  <<Prefix:Prefixlen,Rest/bits>> = Addr,
+  <<Prefix:Prefixlen,1:(bit_size(Rest))>>.
 
 parse_cidr(<<CIDR/binary>>) -> parse_cidr(binary_to_list(CIDR));
 parse_cidr(CIDR) ->
@@ -202,7 +203,7 @@ update_bird_conf(Domains) ->
   Output = os:cmd("ip -j addr"),
   {ok, JSON} = thoas:decode(Output),
   Bridges = maps:from_list([get_cidrs(L) || L <- JSON, startswith(maps:get(<<"ifname">>, L), <<"verlbr">>)]),
-  AddrMap = maps:from_list([{Addr, {bridge_addr(NetAddr), Prefixlen}} || {_, #domain{network_addrs = {NetAddr, Prefixlen}, ipv4_addr = Addr}} <- Domains]),
+  AddrMap = maps:from_list([{Addr, {bridge_addr(NetAddr), Prefixlen}} || {_, #{network_addrs := {NetAddr, Prefixlen}, ipv4_addr := Addr}} <- Domains]),
   AddrToBridgeMap = maps:map(fun (_, Net) -> maps:get(Net, Bridges) end, AddrMap),
 
   io:format("DOMAINS: ~p~n", [Domains]),
@@ -258,9 +259,10 @@ sync_taps(Domains) ->
   {ok, JSONTaps} = thoas:decode(OutputTaps),
   io:format("TAPS: ~p~n", [JSONTaps]),
   TapsActual = sets:from_list([maps:get(<<"ifname">>, L) || L <- JSONTaps, startswith(maps:get(<<"ifname">>, L), <<"verltap">>)]),
-  TapsTarget = sets:from_list([TapName || {_, #domain{tap_name = TapName}} <- Domains]),
-  io:format("Taps to add: ~p~n", [sets:to_list(TapsTarget)]),
-  TapsMap = maps:from_list([{Tap, {MacAddr, network_cidrs_to_bride_cidrs(Cidrs)}} || {_, #domain{network_addrs = Cidrs, tap_name = Tap, mac_addr = MacAddr}} <- Domains]),
+  TapsTarget = sets:from_list([iolist_to_binary(TapName) || {_, #{tap_name := TapName}} <- Domains]), % TODO: persist tap_name as binary
+  io:format("Taps Target: ~p~n", [sets:to_list(TapsTarget)]),
+  io:format("Taps Actual: ~p~n", [sets:to_list(TapsActual)]),
+  TapsMap = maps:from_list([{iolist_to_binary(Tap), {MacAddr, network_cidrs_to_bride_cidrs(Cidrs)}} || {_, #{network_addrs := Cidrs, tap_name := Tap, mac_addr := MacAddr}} <- Domains]),
   io:format("TapsMap: ~p~n", [TapsMap]),
 
   TapsToDelete = sets:subtract(TapsActual, TapsTarget),
@@ -281,8 +283,7 @@ end, sets:to_list(TapsToDelete)),
 add_taps(M) when is_map(M) -> add_taps(maps:to_list(M));
 add_taps([]) -> ok;
 add_taps([{Tap, {Mac, Bridge}}|T]) ->
-  <<A:16, B:16, C:16, D:16, E:16, F:16>> =  binary:encode_hex(Mac),
-  MacAddrString = <<A:16, $::8, B:16, $::8, C:16, $::8, D:16, $::8, E:16, $::8, F:16>>,
+  MacAddrString = virtuerl_util:mac_to_str(Mac),
   Cmd = io_lib:format("ip tuntap add dev ~s mode tap~nip link set dev ~s address ~s master ~s~nip link set ~s up~n", [Tap, Tap, MacAddrString, Bridge, Tap]),
   io:format(Cmd),
   os:cmd(Cmd),