diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 11f79cb9..949b2798 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -110,6 +110,19 @@ jobs: # https://github.com/NixOS/nix/issues/8881 run: nix build --option sandbox false --print-build-logs .#checks.x86_64-linux.lab-restic-test + nix-flake-check-lab-sftpgo: + # Run after flake check + needs: [nix-flake-check-no-build] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.7 + - uses: DeterminateSystems/nix-installer-action@v13 + - uses: DeterminateSystems/magic-nix-cache-action@v7 + - name: nix build + # nix flake check doesn't have a way to specify a specific test to run + # https://github.com/NixOS/nix/issues/8881 + run: nix build --option sandbox false --print-build-logs .#checks.x86_64-linux.lab-sftpgo-test + nix-flake-check-vps-wagtail: # Run after flake check needs: [nix-flake-check-no-build] diff --git a/flake.nix b/flake.nix index f60c27d2..1e677696 100755 --- a/flake.nix +++ b/flake.nix @@ -134,6 +134,7 @@ lab-bitwarden-test = import ./tests/lab-bitwarden.nix checkArgs; lab-immich-test = import ./tests/lab-immich.nix checkArgs; lab-restic-test = import ./tests/lab-restic.nix checkArgs; + lab-sftpgo-test = import ./tests/lab-sftpgo.nix checkArgs; vps-wagtail-test = import ./tests/vps-wagtail.nix checkArgs; }; }; diff --git a/nixos/hosts/lab/default.nix b/nixos/hosts/lab/default.nix index fffdf0af..344ecbe2 100644 --- a/nixos/hosts/lab/default.nix +++ b/nixos/hosts/lab/default.nix @@ -17,6 +17,7 @@ ./immich ./rathole ./restic + ./sftpgo ]; # System76 Pangolin Performance uses BIOS so we need to disable systemd-boot and use grub diff --git a/nixos/hosts/lab/rathole/compose.rathole.sftpgo.yml b/nixos/hosts/lab/rathole/compose.rathole.sftpgo.yml new file mode 100644 index 00000000..ca4bda9a --- /dev/null +++ b/nixos/hosts/lab/rathole/compose.rathole.sftpgo.yml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Andrew Hayzen +# +# SPDX-License-Identifier: MPL-2.0 + +services: + rathole: + links: + - sftpgo diff --git a/nixos/hosts/lab/rathole/default.nix b/nixos/hosts/lab/rathole/default.nix index c53cdc78..7ac46779 100644 --- a/nixos/hosts/lab/rathole/default.nix +++ b/nixos/hosts/lab/rathole/default.nix @@ -13,7 +13,8 @@ ahayzen.docker-compose-files = [ ./compose.rathole.yml ] ++ lib.optional config.ahayzen.lab.actual ./compose.rathole.actual.yml ++ lib.optional config.ahayzen.lab.bitwarden ./compose.rathole.bitwarden.yml - ++ lib.optional config.ahayzen.lab.immich ./compose.rathole.immich.yml; + ++ lib.optional config.ahayzen.lab.immich ./compose.rathole.immich.yml + ++ lib.optional config.ahayzen.lab.sftpgo ./compose.rathole.sftpgo.yml; age.secrets = lib.mkIf (!config.ahayzen.testing) { rathole_toml = { diff --git a/nixos/hosts/lab/rathole/rathole.vm.toml b/nixos/hosts/lab/rathole/rathole.vm.toml index 42b8bb89..2d2db93a 100644 --- a/nixos/hosts/lab/rathole/rathole.vm.toml +++ b/nixos/hosts/lab/rathole/rathole.vm.toml @@ -21,3 +21,6 @@ local_addr = "bitwarden:8080" [client.services.immich] local_addr = "immich-server:3001" + +[client.services.sftpgo] +local_addr = "sftpgo:8080" diff --git a/nixos/hosts/lab/sftpgo/compose.sftpgo.yml b/nixos/hosts/lab/sftpgo/compose.sftpgo.yml new file mode 100644 index 00000000..a3853bfa --- /dev/null +++ b/nixos/hosts/lab/sftpgo/compose.sftpgo.yml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Andrew Hayzen +# +# SPDX-License-Identifier: MPL-2.0 + +services: + sftpgo: + image: docker.io/drakkan/sftpgo:v2.6.2@sha256:05b8e197e796366f955a3816b42a8ed29a9ef400c0da23ecc62bbc22748d4ab8 + environment: + # Allow for connections to continue for 5s before killing + SFTPGO_GRACE_TIME: 5 + # Disable SFTP + SFTPGO_SFTPD__BINDINGS__0__PORT: 0 + expose: + - 8080 + restart: unless-stopped + volumes: + # SFTP backups + - /mnt/mapping-data-user1000/app/sftpgo/backups:/srv/sftpgo/backups + # SFTPGo home + - /mnt/mapping-data-user1000/user:/srv/sftpgo/data + # SFTP settings + - /var/lib/docker-compose-runner-user1000/sftpgo:/var/lib/sftpgo + # Other data + - /mnt/mapping-data-user1000/camera:/mnt/data/camera:ro diff --git a/nixos/hosts/lab/sftpgo/default.nix b/nixos/hosts/lab/sftpgo/default.nix new file mode 100644 index 00000000..3cbc9ba9 --- /dev/null +++ b/nixos/hosts/lab/sftpgo/default.nix @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Andrew Hayzen +# +# SPDX-License-Identifier: MPL-2.0 + +{ config, options, lib, pkgs, ... }: +{ + options.ahayzen.lab.sftpgo = lib.mkOption { + default = true; + type = lib.types.bool; + }; + + config = lib.mkIf (config.ahayzen.lab.sftpgo) { + ahayzen = { + docker-compose-files = [ ./compose.sftpgo.yml ]; + + # Take a snapshot of the database daily + periodic-daily-commands = [ + ''/run/wrappers/bin/sudo --user=unpriv ${pkgs.sqlite}/bin/sqlite3 /var/lib/docker-compose-runner/sftpgo/sftpgo.db ".backup /var/lib/docker-compose-runner/sftpgo/sftpgo-snapshot-$(date +%w).db"'' + ]; + }; + }; +} diff --git a/nixos/hosts/vps/homepage/services.yaml b/nixos/hosts/vps/homepage/services.yaml index feafca7c..10cc7522 100644 --- a/nixos/hosts/vps/homepage/services.yaml +++ b/nixos/hosts/vps/homepage/services.yaml @@ -22,3 +22,9 @@ icon: bitwarden.png description: "The #1 most trusted password manager" siteMonitor: "http://rathole:8080" + + - SFTPGo: + href: https://home.hayzen.com/sftpgo/ + icon: sftpgo.png + description: "Bring your file transfers anywhere" + siteMonitor: "http://rathole:8880" diff --git a/nixos/hosts/vps/rathole/rathole.Caddyfile b/nixos/hosts/vps/rathole/rathole.Caddyfile index d149a9bd..95c753b5 100644 --- a/nixos/hosts/vps/rathole/rathole.Caddyfile +++ b/nixos/hosts/vps/rathole/rathole.Caddyfile @@ -8,6 +8,13 @@ # eg actual IN CNAME ahayzen.com. # +home.hayzen.uk { + # SFTPGo proxy + handle_path /sftpgo/* { + reverse_proxy rathole:8880 + } +} + # Actual proxy actual.ahayzen.com { reverse_proxy rathole:8506 diff --git a/nixos/hosts/vps/rathole/rathole.vm.toml b/nixos/hosts/vps/rathole/rathole.vm.toml index 012fa073..3d1e05b4 100644 --- a/nixos/hosts/vps/rathole/rathole.vm.toml +++ b/nixos/hosts/vps/rathole/rathole.vm.toml @@ -14,3 +14,6 @@ bind_addr = "0.0.0.0:8080" [server.services.immich] bind_addr = "0.0.0.0:8301" + +[server.services.sftpgo] +bind_addr = "0.0.0.0:8880" diff --git a/secrets/rathole_toml.age b/secrets/rathole_toml.age index 8c3618ad..08488af6 100644 Binary files a/secrets/rathole_toml.age and b/secrets/rathole_toml.age differ diff --git a/tests/lab-actual.nix b/tests/lab-actual.nix index 8910d681..24f0e71c 100644 --- a/tests/lab-actual.nix +++ b/tests/lab-actual.nix @@ -89,6 +89,7 @@ immich = false; rathole = true; restic = false; + sftpgo = false; }; }; diff --git a/tests/lab-bitwarden.nix b/tests/lab-bitwarden.nix index 13830aaa..d827d047 100644 --- a/tests/lab-bitwarden.nix +++ b/tests/lab-bitwarden.nix @@ -89,6 +89,7 @@ immich = false; rathole = true; restic = false; + sftpgo = false; }; }; diff --git a/tests/lab-immich.nix b/tests/lab-immich.nix index 6d1071b1..bf773d0f 100644 --- a/tests/lab-immich.nix +++ b/tests/lab-immich.nix @@ -89,6 +89,7 @@ immich = true; rathole = true; restic = false; + sftpgo = false; }; }; diff --git a/tests/lab-restic.nix b/tests/lab-restic.nix index efe422ab..e69eab66 100644 --- a/tests/lab-restic.nix +++ b/tests/lab-restic.nix @@ -28,6 +28,7 @@ immich = false; rathole = false; restic = true; + sftpgo = false; }; }; diff --git a/tests/lab-sftpgo.nix b/tests/lab-sftpgo.nix new file mode 100644 index 00000000..315da825 --- /dev/null +++ b/tests/lab-sftpgo.nix @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Andrew Hayzen +# +# SPDX-License-Identifier: MPL-2.0 + +# +# Test the following +# +# Lab +# - sftpgo +# - rathole +# - backup machines of lab +# +# VPS +# - caddy +# - rathole +# +# Backup +# - backup script + +(import ./lib.nix) { + name = "lab-sftpgo-test"; + nodes = { + vps = { self, pkgs, ... }: { + imports = + [ + self.nixosModules.headlessSystem + ../nixos/hosts/vps/default.nix + ../nixos/users/headless + ]; + + ahayzen = { + testing = true; + + vps = { + rathole = true; + homepage = false; + wagtail-ahayzen = false; + wagtail-yumekasaito = false; + }; + }; + + # Extra packages for the test + environment.systemPackages = [ pkgs.curl ]; + + networking.hosts = { + "127.0.0.1" = [ "actual.ahayzen.com" "bitwarden.ahayzen.com" "home.hayzen.uk" "ahayzen.com" "yumekasaito.com" ]; + }; + + # Preseed host key + services.openssh.hostKeys = [ ]; + environment.etc = { + # Map the test SSH key for backups + "ssh/ssh_host_ed25519_key" = { + mode = "0400"; + source = ./files/test_vps_ssh_id_ed25519; + }; + "ssh/ssh_host_ed25519_key.pub".source = ./files/test_vps_ssh_id_ed25519.pub; + }; + + # Allow test ssh authentication + users.users.headless.openssh.authorizedKeys.keyFiles = [ + ./files/test_backup_ssh_id_ed25519.pub + ./files/test_lab_ssh_id_ed25519.pub + ]; + + # Match VPS specifications + virtualisation = { + cores = 2; + # Increase so we can fit docker images + diskSize = 2 * 1024; + memorySize = 2 * 1024; + }; + }; + + lab = { self, pkgs, ... }: { + imports = + [ + self.nixosModules.headlessSystem + ../nixos/hosts/lab/default.nix + ../nixos/users/headless + ]; + + ahayzen = { + testing = true; + + lab = { + actual = false; + bitwarden = false; + immich = false; + rathole = true; + restic = false; + sftpgo = true; + }; + }; + + # Extra packages for the test + environment.systemPackages = [ pkgs.curl ]; + + networking.hosts = { + # TODO: can we fix the IP addresses of the testing hosts? + "192.168.1.3" = [ "actual.ahayzen.com" "bitwarden.ahayzen.com" "immich.ahayzen.com" "home.hayzen.uk" "ahayzen.com" "yumekasaito.com" ]; + }; + + # Preseed host hey so we can run automatic backups + services.openssh = { + hostKeys = [ ]; + + # Seed known hosts + knownHosts = { + vps = { + extraHostNames = [ "ahayzen.com" ]; + publicKeyFile = ./files/test_vps_ssh_id_ed25519.pub; + }; + }; + }; + environment.etc = { + # Map the test SSH key for backups + "ssh/ssh_host_ed25519_key" = { + mode = "0400"; + source = ./files/test_lab_ssh_id_ed25519; + }; + "ssh/ssh_host_ed25519_key.pub".source = ./files/test_lab_ssh_id_ed25519.pub; + }; + + # Allow test ssh authentication + users.users.headless.openssh.authorizedKeys.keyFiles = [ + ./files/test_backup_ssh_id_ed25519.pub + ]; + + virtualisation = { + cores = 2; + # Increase so we can fit docker images + diskSize = 4 * 1024; + memorySize = 2 * 1024; + }; + }; + + backup = { self, pkgs, ... }: { + environment = { + etc = { + # Map backup and restore scripts + "ahayzen.com/backup.sh".source = ../scripts/backup.sh; + "ahayzen.com/restore.sh".source = ../scripts/restore.sh; + + # Map restore fixtures + "ahayzen.com/restore/fixtures".source = ./fixtures/vps; + + # Map the test SSH key for backups + "ssh/test_ssh_id_ed25519" = { + mode = "0400"; + source = ./files/test_backup_ssh_id_ed25519; + }; + "ssh/test_ssh_id_ed25519.pub".source = ./files/test_backup_ssh_id_ed25519.pub; + }; + + # Extra packages for the test + systemPackages = with pkgs; [ + python3 + rsync + ]; + }; + + services.openssh = { + enable = true; + + # Seed known hosts + knownHosts = { + lab.publicKeyFile = ./files/test_lab_ssh_id_ed25519.pub; + vps = { + extraHostNames = [ "ahayzen.com" ]; + publicKeyFile = ./files/test_vps_ssh_id_ed25519.pub; + }; + }; + }; + + # Setup IdentityFile + programs.ssh.extraConfig = builtins.readFile ./files/ssh_config; + }; + }; + + testScript = '' + import datetime + + start_all() + + labdayofweek = "" + vpsdayofweek = "" + + wait_for_wagtail_cmd = 'journalctl --boot --no-pager --quiet --unit docker.service --grep "\[INFO\] Listening at: http:\/\/0\.0\.0\.0:8080"' + + # + # Test that the VPS boots and shows wagtail admin + # + + with subtest("Ensure docker starts and caddy starts"): + # Wait for docker runner + vps.wait_for_unit("docker-compose-runner", timeout=120) + + # Wait for caddy to start + vps.wait_for_open_port(80, timeout=60) + + # + # Test that Lab works + # + + with subtest("Ensure docker starts"): + lab.wait_for_unit("docker-compose-runner", timeout=120) + + with subtest("Rathole connection"): + # Check we have a server control channel + vps.wait_until_succeeds('journalctl --boot --no-pager --quiet --unit docker.service --grep "rathole::server: Control channel established service=sftpgo"' , timeout=10) + + # Check we have a client control channel + lab.wait_until_succeeds('journalctl --boot --no-pager --quiet --unit docker.service --grep "rathole::client: Control channel established"' , timeout=10) + + with subtest("Test sftpgo"): + # Wait for sftpgo to start + wait_for_sftpgo_cmd = 'journalctl --boot --no-pager --quiet --unit docker.service --grep "starting SFTPGo"' + lab.wait_until_succeeds(wait_for_sftpgo_cmd, timeout=60) + wait_for_sftpgo_cmd = 'journalctl --boot --no-pager --quiet --unit docker.service --grep "server listener registered, address"' + lab.wait_until_succeeds(wait_for_sftpgo_cmd, timeout=60) + + # Test login page + output = vps.succeed("curl --insecure --location --silent home.hayzen.uk/sftpgo/web/admin/setup") + assert "WebAdmin" in output, f"'{output}' does not contain 'WebAdmin'" + + # + # Test that we can backup lab + # + + with subtest("Ensure SSH is ready"): + lab.wait_for_open_port(8022, timeout=30) + + with subtest("Attempt to run lab backup"): + backup.succeed("mkdir -p /tmp/backup-root-lab") + + # Check that the permissions are correct + lab.succeed("ls -nd /var/lib/docker-compose-runner/sftpgo/sftpgo.db | awk 'NR==1 {if ($3 == 2000) {exit 0} else {exit 1}}'") + + # Trigger a snapshot + labdayofweek = datetime.datetime.today().strftime('%w') + lab.succeed("systemctl start periodic-daily.service") + + # Run the backup + backup.succeed("/etc/ahayzen.com/backup.sh lab /etc/ssh/test_ssh_id_ed25519 headless@lab /tmp/backup-root-lab") + + # Check volumes are appearing + backup.succeed("test -d /tmp/backup-root-lab/docker-compose-runner/sftpgo") + + # Check that known files exist and permissions are correct + backup.succeed("test -e /tmp/backup-root-lab/docker-compose-runner/sftpgo/sftpgo-snapshot-" + labdayofweek + ".db") + backup.succeed("ls -nd /tmp/backup-root-lab/docker-compose-runner/sftpgo/sftpgo-snapshot-" + labdayofweek + ".db | awk 'NR==1 {if ($3 == 2000) {exit 0} else {exit 1}}'") + backup.succeed("test -e /tmp/backup-root-lab/docker-compose-runner/sftpgo/sftpgo.db") + backup.succeed("ls -nd /tmp/backup-root-lab/docker-compose-runner/sftpgo/sftpgo.db | awk 'NR==1 {if ($3 == 2000) {exit 0} else {exit 1}}'") + + + # + # Test auto backup in lab + # + # Do this after other backups so that we have snapshots + with subtest("Test Auto Backup Machines"): + # Run backup command + lab.succeed("systemctl start backup-machines.service") + + # + # Check lab is correct + # + + # Check volumes are appearing + lab.succeed("test -d /mnt/data/backup/lab/latest/docker-compose-runner/sftpgo/") + + # Check that known files exist and permissions are correct + lab.succeed("test -e /mnt/data/backup/lab/latest/docker-compose-runner/sftpgo/sftpgo-snapshot-" + labdayofweek + ".db") + lab.succeed("ls -nd /mnt/data/backup/lab/latest/docker-compose-runner/sftpgo/sftpgo-snapshot-" + labdayofweek + ".db | awk 'NR==1 {if ($3 == 2000) {exit 0} else {exit 1}}'") + lab.succeed("test -e /mnt/data/backup/lab/latest/docker-compose-runner/sftpgo/sftpgo.db") + lab.succeed("ls -nd /mnt/data/backup/lab/latest/docker-compose-runner/sftpgo/sftpgo.db | awk 'NR==1 {if ($3 == 2000) {exit 0} else {exit 1}}'") + + with subtest("General metrics (lab)"): + print(lab.succeed("cat /etc/hosts")) + print(lab.succeed("ps auxf")) + print(lab.succeed("free -h")) + print(lab.succeed("df -h")) + print(lab.succeed("docker images")) + print(lab.succeed("docker stats --no-stream")) + + with subtest("General metrics (vps)"): + print(vps.succeed("cat /etc/hosts")) + print(vps.succeed("ps auxf")) + print(vps.succeed("free -h")) + print(vps.succeed("df -h")) + print(vps.succeed("docker images")) + print(vps.succeed("docker stats --no-stream")) + ''; +}