From e7f3b76f50765c99368bc5386ba23e1d07d3e7c6 Mon Sep 17 00:00:00 2001 From: Lin Yinfeng Date: Mon, 2 Oct 2023 23:47:37 +0800 Subject: [PATCH] networking/fw-proxy: refactor --- nixos/modules/misc/ports.nix | 1 + nixos/modules/networking/fw-proxy/default.nix | 235 +++++++++++------- .../networking/fw-proxy/tproxy-cgroup.sh | 10 +- .../networking/fw-proxy/tproxy-clean.sh | 25 -- .../networking/fw-proxy/tproxy-setup.sh | 129 ---------- .../networking/fw-proxy/tproxy-use-pid.sh | 21 -- .../modules/networking/fw-proxy/tproxy-use.sh | 6 +- .../profiles/networking/fw-proxy/default.nix | 28 ++- nixos/profiles/networking/mesh/default.nix | 11 +- nixos/profiles/services/acme/default.nix | 12 +- 10 files changed, 196 insertions(+), 282 deletions(-) delete mode 100644 nixos/modules/networking/fw-proxy/tproxy-clean.sh delete mode 100644 nixos/modules/networking/fw-proxy/tproxy-setup.sh delete mode 100644 nixos/modules/networking/fw-proxy/tproxy-use-pid.sh diff --git a/nixos/modules/misc/ports.nix b/nixos/modules/misc/ports.nix index d349f8134..dabde082f 100644 --- a/nixos/modules/misc/ports.nix +++ b/nixos/modules/misc/ports.nix @@ -53,6 +53,7 @@ proxy-socks = 3141; proxy-mixed = 3142; proxy-tproxy = 3143; + proxy-dns = 3149; clash-controller = 3150; transmission-rpc = 3160; elasticsearch = 3170; diff --git a/nixos/modules/networking/fw-proxy/default.nix b/nixos/modules/networking/fw-proxy/default.nix index 2c6c2354d..7b4ffc9c8 100644 --- a/nixos/modules/networking/fw-proxy/default.nix +++ b/nixos/modules/networking/fw-proxy/default.nix @@ -13,10 +13,7 @@ install -Dm644 $disableProxy $out/bin/disable-proxy install -Dm755 $updateClashUrl $out/bin/update-clash-url install -Dm755 $updateClash $out/bin/update-clash - install -Dm755 $tproxySetup $out/bin/fw-tproxy-setup - install -Dm755 $tproxyClean $out/bin/fw-tproxy-clean install -Dm755 $tproxyUse $out/bin/fw-tproxy-use - install -Dm755 $tproxyUsePid $out/bin/fw-tproxy-use-pid install -Dm755 $tproxyCgroup $out/bin/fw-tproxy-cgroup install -Dm755 $tproxyInterface $out/bin/fw-tproxy-if ''; @@ -46,52 +43,17 @@ mainUrl = config.sops.secrets."clash/main".path; alternativeUrl = config.sops.secrets."clash/alternative".path; }; - tproxySetup = pkgs.substituteAll { - src = ./tproxy-setup.sh; - isExecutable = true; - inherit (pkgs.stdenvNoCC) shell; - inherit (pkgs) iproute2 nftables; - tproxyPort = cfg.mixinConfig.tproxy-port; - inherit - (cfg.tproxy) - fwmark - routingTable - rulePriority - cgroup - nftTable - extraFilterRules - maxCgroupLevel - bypassCgroup - bypassCgroupLevel - ; - allCgroups = lib.concatStringsSep " " cfg.tproxy.allCgroups; - proxiedInterfaces = lib.concatStringsSep " " cfg.tproxy.proxiedInterfaces; - inherit tproxyCgroup tproxyInterface; - }; - tproxyClean = pkgs.substituteAll { - src = ./tproxy-clean.sh; - isExecutable = true; - inherit (pkgs.stdenvNoCC) shell; - inherit (pkgs) iproute2 nftables; - inherit (cfg.tproxy) fwmark cgroup nftTable routingTable; - }; tproxyUse = pkgs.substituteAll { src = ./tproxy-use.sh; isExecutable = true; inherit (pkgs.stdenvNoCC) shell; - inherit tproxyUsePid; - }; - tproxyUsePid = pkgs.substituteAll { - src = ./tproxy-use-pid.sh; - isExecutable = true; - inherit (pkgs.stdenvNoCC) shell; - cgroupPath = "/sys/fs/cgroup/${cfg.tproxy.cgroup}"; + cgroupPath = cfg.tproxy.slice; }; tproxyCgroup = pkgs.substituteAll { src = ./tproxy-cgroup.sh; isExecutable = true; inherit (pkgs.stdenvNoCC) shell; - inherit (cfg.tproxy) nftTable maxCgroupLevel; + inherit (cfg.tproxy) nftTable; }; tproxyInterface = pkgs.substituteAll { src = ./tproxy-interface.sh; @@ -100,6 +62,8 @@ inherit (cfg.tproxy) nftTable; }; }; + + tproxyPort = cfg.mixinConfig.tproxy-port; in with lib; { options.networking.fw-proxy = { @@ -117,17 +81,13 @@ in type = with types; bool; default = false; }; - proxiedInterfaces = mkOption { - type = with types; listOf str; - default = []; - }; routingTable = mkOption { type = with types; int; default = 854; }; fwmark = mkOption { - type = with types; str; - default = "0x356"; + type = with types; int; + default = 854; }; rulePriority = mkOption { type = with types; int; @@ -138,26 +98,18 @@ in # tproxy is a keyword in nft default = "fw-tproxy"; }; - bypassCgroupLevel = mkOption { - type = with types; int; - default = 2; + slice = mkOption { + type = with types; str; + default = "tproxy"; }; - bypassCgroup = mkOption { + bypassSlice = mkOption { type = with types; str; - default = "system.slice/clash.service"; + default = "bypasstproxy"; }; maxCgroupLevel = mkOption { type = with types; int; default = 6; }; - cgroup = mkOption { - type = with types; str; - default = "tproxy.slice"; - }; - allCgroups = mkOption { - type = with types; listOf str; - default = []; - }; extraFilterRules = mkOption { type = with types; lines; default = ""; @@ -250,6 +202,8 @@ in "CAP_NET_BIND_SERVICE" "CAP_NET_ADMIN" ]; + # TODO wait for https://github.com/systemd/systemd/pull/29039 + Slice = lib.mkIf (cfg.tproxy.enable) "${cfg.tproxy.bypassSlice}.slice"; }; wantedBy = ["multi-user.target"]; }; @@ -286,36 +240,149 @@ in (mkIf (cfg.tproxy.enable) { netwokring.routerBasics.enable = true; - systemd.services.fw-tproxy = { - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${scripts}/bin/fw-tproxy-setup"; - ExecStopPost = "${scripts}/bin/fw-tproxy-clean"; - }; - after = ["nftables.service" "clash.service"]; - bindsTo = ["clash.service"]; - wantedBy = ["multi-user.target"]; + systemd.network.config.routeTables = { + fw-tproxy = cfg.tproxy.routingTable; }; - networking.firewall = - if config.networking.nftables.enable - then { - extraInputRules = '' - meta mark ${cfg.tproxy.fwmark} counter accept - ''; - } - else { - extraCommands = '' - ip46tables --append nixos-fw --match mark --mark ${cfg.tproxy.fwmark} --jump nixos-fw-accept - ''; + systemd.network.networks."80-fw-tproxy" = { + matchConfig = { + Name = "lo"; }; + routes = [ + { + routeConfig = { + Destination = "0.0.0.0/0"; + Type = "local"; + Table = cfg.tproxy.routingTable; + }; + } + { + routeConfig = { + Destination = "::/0"; + Type = "local"; + Table = cfg.tproxy.routingTable; + }; + } + ]; + routingPolicyRules = [ + { + routingPolicyRuleConfig = { + Family = "both"; + FirewallMark = cfg.tproxy.fwmark; + Priority = config.routingPolicyPriorities.fw-proxy; + Table = cfg.tproxy.routingTable; + }; + } + ]; + }; + networking.nftables.tables."${cfg.tproxy.nftTable}" = { + family = "inet"; + content = '' + set reserved-ip { + typeof ip daddr + flags interval + elements = { + 10.0.0.0/8, # private + 100.64.0.0/10, # private + 127.0.0.0/8, # loopback + 169.254.0.0/16, # link-local + 172.16.0.0/12, # private + 192.0.0.0/24, # private + 192.168.0.0/16, # private + 198.18.0.0/15, # private + 224.0.0.0/4, # multicast + 255.255.255.255/32 # limited broadcast + } + } - networking.fw-proxy.tproxy.allCgroups = [cfg.tproxy.cgroup]; - passthru.fw-proxy-tproxy-scripts = scripts; + set reserved-ip6 { + typeof ip6 daddr + flags interval + elements = { + ::1/128, # loopback + fc00::/7, # private + fe80::/10 # link-local + } + } - systemd.network.config.routeTables = { - fw-tproxy = cfg.tproxy.routingTable; + set proxied-interfaces { + typeof iif + counter + } + + set cgroups { + type cgroupsv2 + counter + elements = { "${cfg.tproxy.slice}.slice" } + } + + set cgroups-bypass { + type cgroupsv2 + counter + elements = { "${cfg.tproxy.bypassSlice}.slice" } + } + + chain prerouting { + type filter hook prerouting priority mangle; policy accept; + + mark ${toString cfg.tproxy.fwmark} \ + meta l4proto {tcp, udp} \ + tproxy to :${toString tproxyPort} \ + counter \ + accept \ + comment "tproxy and accept marked packets (marked by the output chain)" + + jump filter + + meta l4proto {tcp, udp} \ + iif @proxied-interfaces \ + tproxy to :${toString tproxyPort} \ + mark set ${toString cfg.tproxy.fwmark} \ + counter + } + + chain output { + type route hook output priority mangle; policy accept; + + comment "marked packets will be routed to lo" + + socket cgroupv2 level 1 @cgroups-bypass counter return comment "bypass packets of proxy service" + + jump filter + + ${lib.concatMapStringsSep "\n" (level: "meta l4proto { tcp, udp } socket cgroupv2 level ${toString level} @cgroups meta mark set ${toString cfg.tproxy.fwmark}") + (lib.range 1 cfg.tproxy.maxCgroupLevel)} + } + + chain filter { + # TODO enchilada's kernel does not support fib + # fib daddr type local accept + ip daddr @reserved-ip accept + ip6 daddr @reserved-ip6 accept + + ${cfg.tproxy.extraFilterRules} + } + ''; }; + networking.nftables.preCheckRuleset = '' + sed 's/^.*socket cgroupv2.*$//g' -i ruleset.conf + sed 's/elements = { ".*\.slice" }//g' -i ruleset.conf + ''; + networking.firewall.extraInputRules = '' + meta mark ${toString cfg.tproxy.fwmark} counter accept + ''; + + systemd.slices = { + ${cfg.tproxy.slice} = { + requiredBy = ["nftables.service"]; + before = ["nftables.service"]; + }; + ${cfg.tproxy.bypassSlice} = { + requiredBy = ["nftables.service"]; + before = ["nftables.service"]; + }; + }; + + passthru.fw-proxy-tproxy-scripts = scripts; }) (mkIf cfg.auto-update.enable { diff --git a/nixos/modules/networking/fw-proxy/tproxy-cgroup.sh b/nixos/modules/networking/fw-proxy/tproxy-cgroup.sh index 2473c7f4b..aecb438fa 100644 --- a/nixos/modules/networking/fw-proxy/tproxy-cgroup.sh +++ b/nixos/modules/networking/fw-proxy/tproxy-cgroup.sh @@ -2,7 +2,6 @@ # shellcheck shell=bash nft_table="@nftTable@" -max_level="@maxCgroupLevel@" set -e @@ -23,17 +22,12 @@ case "$action" in list) if [ $# != 1 ]; then usage; fi - for level in $(seq 1 $max_level); do - nft list set inet "$nft_table" cgroups-level"$level" - done + nft list set inet "$nft_table" cgroups ;; add | delete) if [ $# != 2 ]; then usage; fi - IFS='/' read -ra path_arr <<<"$path" - level="${#path_arr[@]}" - nft "$action" element inet "$nft_table" cgroups-level"$level" \ - "{ \"$path\" }" + nft "$action" element inet "$nft_table" cgroups "{ \"$path\" }" ;; *) diff --git a/nixos/modules/networking/fw-proxy/tproxy-clean.sh b/nixos/modules/networking/fw-proxy/tproxy-clean.sh deleted file mode 100644 index 5af84b5d4..000000000 --- a/nixos/modules/networking/fw-proxy/tproxy-clean.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!@shell@ -# shellcheck shell=bash - -export PATH="@iproute2@/bin:@nftables@/bin:$PATH" - -routing_table="@routingTable@" -fwmark="@fwmark@" -cgroup="@cgroup@" -nft_table="@nftTable@" - -set -x - -# delete nftables -nft delete table inet $nft_table - -# delete cgroup -rmdir "/sys/fs/cgroup/$cgroup" - -# delete route tables -ip -4 rule delete fwmark "$fwmark" table "$routing_table" -ip -6 rule delete fwmark "$fwmark" table "$routing_table" -ip -4 route delete local default dev lo table "$routing_table" -ip -6 route delete local default dev lo table "$routing_table" - -true # always success diff --git a/nixos/modules/networking/fw-proxy/tproxy-setup.sh b/nixos/modules/networking/fw-proxy/tproxy-setup.sh deleted file mode 100644 index 51bae77f2..000000000 --- a/nixos/modules/networking/fw-proxy/tproxy-setup.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!@shell@ -# shellcheck shell=bash - -export PATH="@iproute2@/bin:@nftables@/bin:$PATH" - -tproxy_port="@tproxyPort@" -routing_table="@routingTable@" -fwmark="@fwmark@" -rule_priority="@rulePriority@" -nft_table="@nftTable@" -bypass_cgroup="@bypassCgroup@" -bypass_cgroup_level="@bypassCgroupLevel@" -max_level="@maxCgroupLevel@" -cgroup="@cgroup@" -all_cgroups=(@allCgroups@) -proxied_interfaces=(@proxiedInterfaces@) - -set -ex - -# setup route tables -ip -4 route add local default dev lo table "$routing_table" -ip -6 route add local default dev lo table "$routing_table" -ip -4 rule add fwmark "$fwmark" table "$routing_table" priority "$rule_priority" -ip -6 rule add fwmark "$fwmark" table "$routing_table" priority "$rule_priority" - -# setup cgroup -mkdir "/sys/fs/cgroup/$cgroup" - -# setup nftables -nft --file - <&2 - exit 1 -fi - -echo "$1" >"$cgroup_path/cgroup.procs" diff --git a/nixos/modules/networking/fw-proxy/tproxy-use.sh b/nixos/modules/networking/fw-proxy/tproxy-use.sh index 4d0515d59..221926939 100644 --- a/nixos/modules/networking/fw-proxy/tproxy-use.sh +++ b/nixos/modules/networking/fw-proxy/tproxy-use.sh @@ -1,8 +1,6 @@ #!@shell@ # shellcheck shell=bash -tproxy_use_pid="@tproxyUsePid@" +cgroup_path="@cgroupPath@" -set -e -sudo "$tproxy_use_pid" $$ >/dev/null 2>&1 -exec "$@" +exec systemd-run --pipe --slice="$cgroup_path" "$@" diff --git a/nixos/profiles/networking/fw-proxy/default.nix b/nixos/profiles/networking/fw-proxy/default.nix index 607f2a91f..518cfc6e9 100644 --- a/nixos/profiles/networking/fw-proxy/default.nix +++ b/nixos/profiles/networking/fw-proxy/default.nix @@ -4,17 +4,43 @@ ... }: let cfg = config.networking.fw-proxy; + mixinCfg = cfg.mixinConfig; inherit (config.networking) hostName; in { networking.fw-proxy = { enable = true; tproxy = { enable = lib.mkDefault true; - cgroup = "tproxy.slice"; + slice = "tproxy"; + bypassSlice = "bypasstproxy"; routingTable = config.routingTables.fw-proxy; rulePriority = config.routingPolicyPriorities.fw-proxy; }; mixinConfig = { + ipv6 = true; + dns = { + enable = true; + listen = "[::]:${toString config.ports.proxy-dns}"; + ipv6 = true; + default-nameserver = [ + "223.5.5.5" + "223.6.6.6" + "[2400:3200::1]:53" + "[2400:3200:baba::1]:53" + ]; + nameserver = [ + "https://101.6.6.6:8443/dns-query" + "https://dns.alidns.com/dns-query" + ]; + fallback = [ + "https://1.1.1.1/dns-query" + "https://dns.google/dns-query" + ]; + failback-filter = { + geoio = true; + geoip-code = "CN"; + }; + }; port = config.ports.proxy-http; socks-port = config.ports.proxy-socks; mixed-port = config.ports.proxy-mixed; diff --git a/nixos/profiles/networking/mesh/default.nix b/nixos/profiles/networking/mesh/default.nix index 89391c038..79f137bd9 100644 --- a/nixos/profiles/networking/mesh/default.nix +++ b/nixos/profiles/networking/mesh/default.nix @@ -39,13 +39,12 @@ in { rtt cost 1024; rtt max 1024 ms; ''; - extraInterfaces = - lib.optionalAttrs config.services.zerotierone.enable { - "${config.passthru.zerotierInterfaceName}" = { - type = "tunnel"; - extraConfig = cfg.bird.babelInterfaceConfig; - }; + extraInterfaces = lib.optionalAttrs config.services.zerotierone.enable { + "${config.passthru.zerotierInterfaceName}" = { + type = "tunnel"; + extraConfig = cfg.bird.babelInterfaceConfig; }; + }; }; sops.secrets."ike_private_key_pem" = { sopsFile = config.sops-file.terraform; diff --git a/nixos/profiles/services/acme/default.nix b/nixos/profiles/services/acme/default.nix index 3d0950963..d0566ad84 100644 --- a/nixos/profiles/services/acme/default.nix +++ b/nixos/profiles/services/acme/default.nix @@ -8,13 +8,17 @@ defaults = { email = "lin.yinfeng@outlook.com"; dnsProvider = "cloudflare"; - dnsResolver = "1.1.1.1:53"; credentialsFile = config.sops.templates.acme-credentials.path; }; }; - systemd.services.acme-main.environment = - lib.mkIf (config.networking.fw-proxy.enable) - config.networking.fw-proxy.environment; + systemd.services.acme-main = lib.mkIf (config.networking.fw-proxy.enable && config.networking.fw-proxy.tproxy.enable) { + # tproxy is needed because + # lego checks all authoritative DNS server + serviceConfig = { + # TODO wait for https://github.com/systemd/systemd/pull/29039 + Slice = "${config.networking.fw-proxy.tproxy.slice}.slice"; + }; + }; sops.secrets."cloudflare_token" = { sopsFile = config.sops-file.get "terraform/common.yaml"; restartUnits = []; # no need to restart units