fix(quotas): make them work for good by using proper kernels and FS opts

For reasons unbeknownst to me, the Oracle Cloud kernel images stopped
shipping the quota kernel modules that are part of more standard Ubuntu
image packages, rendering storage quotas non-functional for a while
(this is something I tried to work around in the past by pinning the
kernel image to a known-good version, in case it was a temporary fluke
in how the package was generated, but it looks like it was not).

Moreover, using external quota files as we were doing is deprecated
(https://askubuntu.com/questions/1492000/deprecation-warning-for-quota-how-to-fix),
but the alternative of enabling the ext4 filesystem quota option was
very inconvenient, as such filesystem was the root filesystem, which
cannot be unmounted from the instance itself while it's booted to change
it.

Thanks to the power of testing stuff with sacrificial instances and VMs,
which took a while in big part due to Oracle's available capacity for
these being low, I now feel confident enough to use a initramfs boot
script to set the desired root filesystem options during early boot
stages, before it's mounted there for good. I also feel confident enough
to use generic kernel images with the required modules. Therefore,
storage quota issues should now be fixed for good, finally 🎉

Management commands cheatsheet.md

@@ -25,3 +25,12 @@ startup of `systemd` user daemons.
 ```bash
 $ sudo machinectl shell <USER>@ [ABSOLUTE PATH TO COMMAND] [PARAMETERS...]
 ```
+
+## View group disk quota status
+
+Checking disk quotas before or after operations that are likely to significantly
+impact disk space usage can be helpful.
+
+```bash
+$ sudo repquota -ga
+```

aylas-one.ansible.yml

@@ -61,7 +61,6 @@
 
     - role: storage_quotas
       vars:
-        pin_oracle_kernel_version: false
         group_quotas:
           - name: khron
             limits: 10G 12G 0 0

roles/storage_quotas/files/rootfsquota_initramfs_hook

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copies the tune2fs binary to the initramfs, so that other initramfs
+# scripts can change root ext4 filesystem options before they get mounted.
+#
+# See man 7 initramfs-tools, /usr/share/initramfs-tools/init and
+# /usr/share/initramfs-tools/scripts/local for more information.
+
+PREREQ=""
+prereqs()
+{
+  echo "$PREREQ"
+}
+
+case $1 in
+prereqs)
+  prereqs
+  exit 0
+  ;;
+esac
+
+# shellcheck disable=SC1091
+. /usr/share/initramfs-tools/hook-functions
+
+copy_exec /sbin/tune2fs

roles/storage_quotas/files/rootfsquota_initramfs_script

@@ -0,0 +1,46 @@
+#!/bin/sh
+# Enables ext4 quota options on the root filesystem, by temporarily
+# unmounting and remounting it shortly after it was mounted by the
+# initramfs init script. This won't work if e.g. /usr or /etc were
+# already mounted on that root filesystem when local-bottom scripts
+# are run, but experimentation found this technique more reliable
+# overall for our purposes than using a local premount hook.
+#
+# See man 7 initramfs-tools, /usr/share/initramfs-tools/init and
+# /usr/share/initramfs-tools/scripts/local for more information.
+
+PREREQ=""
+prereqs()
+{
+  echo "$PREREQ"
+}
+
+case $1 in
+prereqs)
+  prereqs
+  exit 0
+  ;;
+esac
+
+# shellcheck disable=SC1091
+. /scripts/functions
+# shellcheck disable=SC1091
+. /scripts/local # Import mountroot function
+
+while read -r mount; do
+  device="${mount%% *}"
+  mount_point="${mount#* }"; mount_point="${mount_point%% *}"
+
+  # shellcheck disable=SC2154 # rootmnt is always defined in initramfs boot scripts
+  if [ "$mount_point" != "$rootmnt" ]; then
+    continue
+  fi
+
+  echo "Found root filesystem device $device mount, remounting it with ext4 quota options enabled"
+
+  umount "$rootmnt"
+  tune2fs -Q grpquota "$device"
+  mountroot
+
+  break
+done < /proc/mounts

roles/storage_quotas/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: Regenerate initial ramdisk image
+  become: true
+  changed_when: true
+  ansible.builtin.command: update-initramfs -u

roles/storage_quotas/tasks/main.yml

@@ -12,24 +12,57 @@
     src: LABEL=cloudimg-rootfs
     fstype: ext4
     path: /
-    opts: discard,errors=remount-ro,jqfmt=vfsv1,grpjquota=group.quota
+    opts: discard,errors=remount-ro
     state: present
 
-- name: Prevent Oracle kernel updates to ensure quota modules compatibility
+- name: Copy initial ramdisk root filesystem option tweak hook script
   become: true
-  when: pin_oracle_kernel_version
-  loop:
-    - linux-oracle
-    - linux-image-oracle
-    - linux-headers-oracle
-  ansible.builtin.dpkg_selections:
-    name: "{{ item }}"
-    selection: hold
+  notify:
+    - Regenerate initial ramdisk image
+    - Reboot
+  register: rootfsquota_initramfs_hook
+  ansible.builtin.copy:
+    src: rootfsquota_initramfs_hook
+    dest: /etc/initramfs-tools/hooks/rootfsquota
+    mode: u=rwx,g=rx,o=rx
+
+- name: Copy initial ramdisk root filesystem option tweak boot script
+  become: true
+  notify:
+    - Regenerate initial ramdisk image
+    - Reboot
+  register: rootfsquota_initramfs_script
+  ansible.builtin.copy:
+    src: rootfsquota_initramfs_script
+    dest: /etc/initramfs-tools/scripts/local-bottom/rootfsquota
+    mode: u=rwx,g=rx,o=rx
+
+# Swapping the kernel image with the generic one is needed because the Oracle kernel does
+# not reliably ship modules needed for filesystem quota support. In the past they used to,
+# but any security advantage we could get from the smaller image is thus offset by the need
+# to keep the kernel image version pinned. See:
+# https://patchwork.ozlabs.org/project/linux-ext4/patch/[email protected]/
+# https://cateee.net/lkddb/web-lkddb/QFMT_V2.html
+- name: Remove Oracle kernel package
+  become: true
+  ansible.builtin.apt:
+    package: linux-oracle
+    state: absent
+    purge: true
+    autoremove: true
+
+- name: Install generic Ubuntu kernel package
+  become: true
+  ansible.builtin.apt:
+    package: linux-generic
+    install_recommends: false
 
 # If a reboot is pending due to changing filesystem mount flags,
 # we need to do it now to be able to configure quotas
 - name: Flush pending handlers
-  when: storage_quotas_quota_flags.changed # noqa: no-handler
+  when: storage_quotas_quota_flags.changed or
+    rootfsquota_initramfs_hook.changed or
+    rootfsquota_initramfs_script.changed # noqa: no-handler
   ansible.builtin.meta: flush_handlers
 
 - name: Set up quota files