lab: add automatic restic offsite backup

Closes #142

nixos/hosts/lab/restic/compose.restic-offsite.yml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Andrew Hayzen <[email protected]>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+services:
+  restic-offsite:
+    container_name: restic_offsite
+    image: docker.io/restic/restic:0.17.1@sha256:424a4e1fcc6fe2557b5614239dc71a2c793acb33a83ea217171bd7edc1862dcb
+    entrypoint:
+      - /bin/sh
+    # Check if a repo exists
+    # If it doesn't create one
+    # Then wait forever to keep the container running so we can exec on it
+    command: ["-c", "(/usr/bin/restic cat config || /usr/bin/restic init) && tail -F /dev/null"]
+    env_file:
+      - /etc/restic/offsite.env
+    environment:
+      RESTIC_PASSWORD_FILE: /etc/restic/password
+      XDG_CACHE_HOME: /var/cache
+    # Set a host to ensure that caches and settings are correct in restic
+    # as docker containers have a new host name every time
+    hostname: restic-offsite
+    # Allocate a tty then restic prints more info used for testing
+    tty: true
+    volumes:
+      - /etc/restic/offsite_env:/etc/restic/offsite_env:ro
+      - /etc/restic/password:/etc/restic/password:ro
+      # The source data
+      - /mnt/data:/mnt/data:ro
+      # Cache uses XDG_CACHE_HOME/restic
+      # https://restic.readthedocs.io/en/stable/manual_rest.html#caching
+      - /var/cache/docker-compose-runner/restic:/var/cache/restic:rw

nixos/hosts/lab/restic/default.nix

@@ -11,7 +11,7 @@
 
   config = lib.mkIf (config.ahayzen.lab.restic) {
     ahayzen = {
-      docker-compose-files = [ ./compose.restic-local.yml ];
+      docker-compose-files = [ ./compose.restic-local.yml ] ++ lib.optional (!config.ahayzen.testing) ./compose.restic-offsite.yml;
 
       # Take a snapshot of the data daily (do not scan as this is providing estimates)
       # Keep daily snapshots for the last week, weekly for the last month, monthly for the last year, and yearly for the last 5 years
@@ -24,14 +24,26 @@
           /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_local /bin/sh -c "/usr/bin/restic snapshots"
           /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_local /bin/sh -c "/usr/bin/restic stats --mode files-by-contents"
         ''
-      ];
+      ] ++ lib.optional (!config.ahayzen.testing)
+        ''
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic backup --host restic-offsite --no-scan --quiet /mnt/data"
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic forget --keep-within-daily 7d --keep-within-weekly 1m --keep-within-monthly 1y --keep-within-yearly 5y"
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic prune"
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic snapshots"
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic stats --mode files-by-contents"
+        ''
+      ;
 
-      # Check 5% of the local data weekly, so all data should be checked in around 20 weeks
+      # Check 5% of the data weekly, so all data should be checked in around 20 weeks
       periodic-weekly-commands = [
         ''
           /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_local /bin/sh -c "/usr/bin/restic check --read-data-subset=5%"
         ''
-      ];
+      ] ++ lib.optional (!config.ahayzen.testing)
+        ''
+          /run/wrappers/bin/sudo ${pkgs.docker}/bin/docker exec -t restic_offsite /bin/sh -c "/usr/bin/restic check --read-data-subset=5%"
+        ''
+      ;
     };
 
     age.secrets = lib.mkIf (!config.ahayzen.testing) {
@@ -42,13 +54,35 @@
         owner = "unpriv";
         group = "unpriv";
       };
+
+      restic_offsite_env = {
+        file = ../../../../secrets/restic_offsite_env.age;
+        # Set correct owner otherwise docker cannot read the file
+        mode = "0600";
+        owner = "unpriv";
+        group = "unpriv";
+      };
     };
 
     environment.etc = {
       "restic/password".source =
         if config.ahayzen.testing
         then ./password.vm
         else config.age.secrets.restic_password.path;
+      "restic/offsite.env".source =
+        if config.ahayzen.testing
+        then ./offsite.env
+        else config.age.secrets.restic_offsite_env.path;
     };
+
+    # Restart if static files change
+    #
+    # Note agenix files are not possible and will need the version bumping
+    # which causes the hash of the docker-compose file to change.
+    systemd.services."docker-compose-runner".restartTriggers = [
+      # Agenix path with a version that can be bumped
+      "/etc/restic/password"
+      "/etc/restic/offsite.env"
+    ];
   };
 }

nixos/hosts/lab/restic/offsite.env

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=aaaaaa
+AWS_SECRET_ACCESS_KEY=bbbbbb
+RESTIC_REPOSITORY=s3:domain/bucket

nixos/hosts/lab/restic/offsite.env.license

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: Andrew Hayzen <[email protected]>
+#
+# SPDX-License-Identifier: MPL-2.0

secrets/restic_offsite_env.age

@@ -0,0 +1,19 @@
+age-encryption.org/v1
+-> ssh-ed25519 q0BJOw pnhOh5lAPVeq+N8ajGuKDqX5LtzQMx3YVz+RXK0sPAY
+fC5ZIYYI0Lxm3y+ymdBeG0NWxXUjdzFdRyx/H3raDZo
+-> ssh-ed25519 2sBEqQ PxVw224Ra1lC2RpJDgwX4MYx3/pcokLQ5fHMOLhyRVw
+2lHDCaGaHcsshGnRIec5V+QSxPGrKPBjBf/xkmgKxBY
+-> ssh-rsa xfbDHA
+qFk1j01++EnfevmPqbEkLNmlR1d5P2e8X4dU0Zqb+ekJnrfuwYcLoayV7+U6zFtK
+eD/Ko3fosxgKYN5Dy8Hn+08dWHZS3pMoMfCqJRyBGdLk/VLWZzIt7BremhzFlIj6
+CKOA6TZtJ/RG7yAVsilVHsuqcwKOr+gQ86fN0QKZjGaLGpcDfI/0EEx2QM85izqt
+hAGmUhc0fslWiOpNCwz5TDlXAS5XK3iZjtM/zOzoGKif/qHLje7aYAvYKyJZkKXM
+X1Scq2TG/s3+TM/8V+aV79JHB8fCuJ9z38IG5q2bGBEci2tdsIhpqJ/EhLXUEPR0
+wUwq1aqgyKm7ZVE3jkmO5PIYP66rxla8mrNWfDGFoBQW5QpKCuLTgBMC4yJWMkbF
+q4Ce1xS0oK8RgGSWjCs4FeTx4Nyl0N3yulNntcBVROiStL/Yg/Z+qOsjpz70vzsZ
+/Bv7XLej9VS6CfKegfvjaPMzUQ06/H5ciEZ3W0xyd7W1O5cNZyN+5MtoyP9R7PSu
+qANI66SFCELPNBleo5GdvuJ8BmGSDJkA3ho4L+wIN4vKG/tNIcbAeIB/3FoYNMg1
+q7H6o+ZPpM5GP+pkBIqsN3942I2VU05QRFkiNNkBVKT99wfBBVsu1jnod7waKmL9
+ugBTcFqzETUw9k9NEqyYsWCZ5hBeCuq3Bd0ZSA5FPPA
+--- dqoyYSoKga19+P8ZBQifPLkFbSytoio0RlfzGM/1mFM
+ȉbX�t���F��b޶�%�HF�nj�\VP��f�a��яf�#KEAtg���CܟlC!w����M�Jш��{/�W6��ذ��f�_���"�[D��v�!�OVx�L�gpmg���q^ϣ��c�2�>K��g����}BMs�aB	�}������P����gq�P�\d����d�&tÖ�*��@~̂��OO��"y��hL����5����9sG�����ɚ�3ǻE��@�/���L=V�c<Q�C����Ǹ����t[2��ޕ�ZI��V3����"�!iI_*��,�W��}f�T��U�����v�WOg�ߑ��WX�

secrets/secrets.nix

@@ -12,6 +12,7 @@ in
   "bitwarden_env.age".publicKeys = [ publicKeys.host.lab ] ++ publicKeys.group.users.developer;
   "immich_env.age".publicKeys = [ publicKeys.host.lab ] ++ publicKeys.group.users.developer;
   "rathole_toml.age".publicKeys = [ publicKeys.host.lab publicKeys.host.vps ] ++ publicKeys.group.users.developer;
+  "restic_offsite_env.age".publicKeys = [ publicKeys.host.lab ] ++ publicKeys.group.users.developer;
   "restic_password.age".publicKeys = [ publicKeys.host.lab ] ++ publicKeys.group.users.developer;
 
   # For avoiding new lines use