initial commit

.sops.yaml

@@ -0,0 +1,11 @@
+---
+creation_rules:
+  - path_regex: kubernetes/.*\.sops\.ya?ml
+    encrypted_regex: "^(data|stringData)$"
+    key_groups:
+      - age:
+          - "age12rzrdtn8xhd89y23qw4kymxftuylqn5cm522jcn327atent4a40swjcgmj"
+  - path_regex: ansible/.*\.sops\.ya?ml
+    key_groups:
+      - age:
+          - "age12rzrdtn8xhd89y23qw4kymxftuylqn5cm522jcn327atent4a40swjcgmj"

ansible/.ansible-lint

@@ -0,0 +1,9 @@
+skip_list:
+  - yaml[commas]
+  - yaml[line-length]
+  - var-naming
+warn_list:
+  - command-instead-of-shell
+  - deprecated-command-syntax
+  - experimental
+  - no-changed-when

ansible/inventory/group_vars/controllers/main.yaml

@@ -0,0 +1,28 @@
+---
+k3s_control_node: true
+k3s_server:
+  cluster-cidr: "10.69.0.0/16"
+  service-cidr: "10.96.0.0/16"
+  disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
+  disable-cloud-controller: true
+  disable-kube-proxy: true
+  disable-network-policy: true
+  docker: false
+  embedded-registry: true
+  etcd-expose-metrics: true
+  flannel-backend: "none"
+  kube-apiserver-arg:
+    - "anonymous-auth=true"
+  kube-controller-manager-arg:
+    - "bind-address=0.0.0.0"
+  kube-scheduler-arg:
+    - "bind-address=0.0.0.0"
+  kubelet-arg:
+    - "image-gc-high-threshold=55"
+    - "image-gc-low-threshold=50"
+  node-ip: "{{ ansible_host }}"
+  pause-image: registry.k8s.io/pause:3.9
+  secrets-encryption: true
+  tls-san:
+    - "192.168.1.200"
+  write-kubeconfig-mode: "644"

ansible/inventory/group_vars/kubernetes/main.yaml

@@ -0,0 +1,23 @@
+---
+k3s_become: true
+k3s_etcd_datastore: true
+k3s_install_hard_links: true
+k3s_registration_address: "192.168.1.200"
+k3s_registries:
+  mirrors:
+    docker.io:
+    gcr.io:
+    ghcr.io:
+    k8s.gcr.io:
+    lscr.io:
+    mcr.microsoft.com:
+    public.ecr.aws:
+    quay.io:
+    registry.k8s.io:
+# renovate: datasource=github-releases depName=k3s-io/k3s
+k3s_release_version: v1.29.1+k3s2
+k3s_server_manifests_templates:
+  - custom-cilium-helmchart.yaml
+  - kube-vip-ds.yaml
+  - kube-vip-rbac.yaml
+k3s_use_unsupported_config: true

ansible/inventory/hosts.yaml

@@ -0,0 +1,14 @@
+---
+kubernetes:
+  children:
+    controllers:
+      hosts:
+        "odroid-01":
+          ansible_user: "durden"
+          ansible_host: "192.168.1.201"
+        "odroid-02":
+          ansible_user: "durden"
+          ansible_host: "192.168.1.202"
+        "odroid-03":
+          ansible_user: "durden"
+          ansible_host: "192.168.1.203"

ansible/playbooks/cluster-installation.yaml

@@ -0,0 +1,60 @@
+---
+- name: Cluster Installation
+  hosts: kubernetes
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Check if cluster is installed
+      check_mode: false
+      ansible.builtin.stat:
+        path: /etc/rancher/k3s/config.yaml
+      register: k3s_installed
+
+    - name: Ignore manifests templates and urls if the cluster is already installed
+      when: k3s_installed.stat.exists
+      ansible.builtin.set_fact:
+        k3s_server_manifests_templates: []
+        k3s_server_manifests_urls: []
+
+    - name: Prevent downgrades
+      when: k3s_installed.stat.exists
+      ansible.builtin.include_tasks: tasks/version-check.yaml
+
+    - name: Install Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: installed
+
+    - name: Kubeconfig
+      ansible.builtin.include_tasks: tasks/kubeconfig.yaml
+
+    - name: Wait for custom manifests to rollout
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{{ item.kind }}"
+        name: "{{ item.name }}"
+        namespace: "{{ item.namespace | default('') }}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
+
+    - name: Cilium
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cilium.yaml
+
+    - name: Cruft
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cruft.yaml

ansible/playbooks/cluster-kube-vip.yaml

@@ -0,0 +1,24 @@
+---
+- name: Cluster kube-vip
+  hosts: controllers
+  serial: 1
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Ensure Kubernetes is running
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: started
+
+    - name: Upgrade kube-vip
+      ansible.builtin.template:
+        src: templates/kube-vip-ds.yaml
+        dest: "{{ k3s_server_manifests_dir }}/kube-vip-ds.yaml"
+        mode: preserve

ansible/playbooks/cluster-nuke.yaml

@@ -0,0 +1,101 @@
+---
+- name: Cluster Nuke
+  hosts: kubernetes
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  vars_prompt:
+    - name: nuke
+      prompt: |-
+        Are you sure you want to nuke this cluster?
+        Type 'YES I WANT TO DESTROY THIS CLUSTER' to proceed
+      default: "n"
+      private: false
+  pre_tasks:
+    - name: Check for confirmation
+      ansible.builtin.fail:
+        msg: Aborted nuking the cluster
+      when: nuke != 'YES I WANT TO DESTROY THIS CLUSTER'
+
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Stop Kubernetes # noqa: ignore-errors
+      ignore_errors: true
+      block:
+        - name: Stop Kubernetes
+          ansible.builtin.include_role:
+            name: xanmanning.k3s
+            public: true
+          vars:
+            k3s_state: stopped
+
+    # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
+    - name: Networking
+      block:
+        - name: Networking | Delete Cilium links
+          ansible.builtin.command:
+            cmd: "ip link delete {{ item }}"
+            removes: "/sys/class/net/{{ item }}"
+          loop: ["cilium_host", "cilium_net", "cilium_vxlan"]
+        - name: Networking | Flush iptables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Flush ip6tables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+            ip_version: ipv6
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Delete CNI directory
+          ansible.builtin.file:
+            path: /etc/cni/net.d
+            state: absent
+
+    - name: Check to see if k3s-killall.sh exits
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s-killall.sh
+      register: check_k3s_killall_script
+
+    - name: Check to see if k3s-uninstall.sh exits
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s-uninstall.sh
+      register: check_k3s_uninstall_script
+
+    - name: Run k3s-killall.sh
+      when: check_k3s_killall_script.stat.exists
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-killall.sh
+      register: k3s_killall
+      changed_when: k3s_killall.rc == 0
+
+    - name: Run k3s-uninstall.sh
+      when: check_k3s_uninstall_script.stat.exists
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-uninstall.sh
+      args:
+        removes: /usr/local/bin/k3s-uninstall.sh
+      register: k3s_uninstall
+      changed_when: k3s_uninstall.rc == 0
+
+    - name: Ensure hard links are removed
+      when:
+        - k3s_install_hard_links
+        - not ansible_check_mode
+      ansible.builtin.file:
+        path: "{{ k3s_install_dir }}/{{ item }}"
+        state: absent
+      loop: ["kubectl", "crictl", "ctr"]
+
+    - name: Remove local storage path
+      ansible.builtin.file:
+        path: /var/openebs/local
+        state: absent
+
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting hosts
+        reboot_timeout: 3600

ansible/playbooks/cluster-prepare.yaml

@@ -0,0 +1,112 @@
+---
+- name: Prepare System
+  hosts: kubernetes
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+    - name: Populate service facts
+      ansible.builtin.service_facts:
+  tasks:
+    - name: Locale
+      block:
+        - name: Locale | Set timezone
+          community.general.timezone:
+            name: "America/Denver"
+
+    - name: Packages
+      block:
+        - name: Packages | Install
+          ansible.builtin.apt:
+            name: apt-transport-https,ca-certificates,conntrack,curl,dirmngr,gdisk,gnupg,hdparm,htop,
+              iptables,iputils-ping,ipvsadm,libseccomp2,lm-sensors,net-tools,nfs-common,
+              nvme-cli,open-iscsi,parted,psmisc,python3,python3-apt,python3-kubernetes,python3-yaml,
+              smartmontools,socat,software-properties-common,unzip,util-linux
+            install_recommends: false
+
+    - name: Network Configuration
+      notify: Reboot
+      block:
+        - name: Network Configuration | Set hostname
+          ansible.builtin.hostname:
+            name: "{{ inventory_hostname }}"
+        - name: Network Configuration | Update hosts
+          ansible.builtin.copy:
+            content: |
+              127.0.0.1 localhost
+              127.0.1.1 {{ inventory_hostname }}
+
+              # The following lines are desirable for IPv6 capable hosts
+              ::1     localhost ip6-localhost ip6-loopback
+              ff02::1 ip6-allnodes
+              ff02::2 ip6-allrouters
+            dest: /etc/hosts
+            mode: preserve
+        # https://github.com/onedr0p/cluster-template/discussions/635
+        - name: Network Configuration | Remove immutable flag from /etc/resolv.conf
+          ansible.builtin.file:
+            attributes: -i
+            path: /etc/resolv.conf
+        - name: Network Configuration | Remove /etc/resolv.conf
+          ansible.builtin.file:
+            attributes: -i
+            path: /etc/resolv.conf
+            state: absent
+        - name: Network Configuration | Add custom /etc/resolv.conf
+          ansible.builtin.copy:
+            attributes: +i
+            mode: '0644'
+            dest: /etc/resolv.conf
+            content: |
+              search .
+              nameserver 1.1.1.1
+              nameserver 8.8.8.8
+
+    - name: System Configuration
+      notify: Reboot
+      block:
+        - name: System Configuration | Disable apparmor
+          when: ansible_facts.services['apparmor.service'] is defined
+          ansible.builtin.systemd:
+            name: apparmor
+            state: stopped
+            masked: true
+        - name: System Configuration | Disable swap
+          ansible.posix.mount:
+            name: "{{ item }}"
+            fstype: swap
+            state: absent
+          loop: ["none", "swap"]
+        - name: System Configuration | Create Kernel modules
+          ansible.builtin.copy:
+            dest: "/etc/modules-load.d/{{ item }}.conf"
+            mode: "0644"
+            content: "{{ item }}"
+          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "nbd", "overlay", "rbd"]
+          register: modules_status
+        - name: System Configuration | Reload Kernel modules # noqa: no-changed-when no-handler
+          when: modules_status.changed
+          ansible.builtin.systemd:
+            name: systemd-modules-load
+            state: restarted
+        - name: System Configuration | Sysctl
+          ansible.posix.sysctl:
+            name: "{{ item.key }}"
+            value: "{{ item.value }}"
+            sysctl_file: /etc/sysctl.d/99-kubernetes.conf
+            reload: true
+          with_dict: "{{ sysctl_config }}"
+          vars:
+            sysctl_config:
+              fs.inotify.max_queued_events: 65536
+              fs.inotify.max_user_watches: 524288
+              fs.inotify.max_user_instances: 8192
+
+  handlers:
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting hosts
+        reboot_timeout: 3600