pixiecore: initial API mode deployment(#408)

the API is satisfied by a Bash script running under fcgiwrap and exposed
by nginx, in a sidecar container.

the script uses kubectl to lookup which Node is trying to boot, based on
a node label containing the MAC address. the script also decides whether
a boot is allowed, and which ignition configs to embed, based on
annotations on the Node.

the ignition configs that can be embedded are the ones that live in
ignition/ ... the pod decrypts these on boot with an age key that is
used only for this porpoise.

the script expects to run with hostPath access to a control-plane node,
so that it can generate join tokens and upload control-plane certs.

cluster/config/kustomizeconfig.yaml

@@ -1,6 +1,8 @@
 nameReference:
-- kind: ConfigMap
-  version: v1
-  fieldSpecs:
-  - path: spec/valuesFrom/name
-    kind: HelmRelease
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
+      - path: /spec/values/additionalVolumes/configMap/name
+        kind: HelmRelease

cluster/kube-system/kustomization.yaml

@@ -11,6 +11,11 @@ configMapGenerator:
       immutable: true
       labels:
         grafana_dashboard: '1'
+  - name: pixiecore-api-scripts
+    files:
+      - pixiecore-api/boot-request.sh
+      - pixiecore-api/generate-ignition.sh
+      - pixiecore-api/nginx.conf
 resources:
   - ../config
   - cert-manager.yaml
@@ -25,6 +30,7 @@ resources:
   - node-feature-discovery.yaml
   - node-problem-detector.yaml
   - nodes.yaml
+  - pixiecore.yaml
   - plans/kube-vip.yaml
   - plans/kubeadm-upgrade-apply.yaml
   - plans/rebase-image.yaml

cluster/kube-system/nodes.yaml

@@ -2,6 +2,11 @@ apiVersion: v1
 kind: Node
 metadata:
   name: m715q-1
+  metadata:
+    annotations:
+      samcday.com/boot-profiles: base,var,tpm
+    labels:
+      samcday.com/mac: 6c-4b-90-17-1e-be
 spec:
   taints:
     # This is a very underpowered machine that is mostly here to serve

cluster/kube-system/pixiecore-api/boot-request.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -ueo pipefail
+
+export KUBECONFIG=/etc/kubernetes/admin.conf
+
+kubectl="chroot /host kubectl"
+
+mac_addr=$(basename "$REQUEST_URI" | tr '[:upper:]' '[:lower:]' | tr ':' '-')
+node=$($kubectl get node -o name -l "samcday.com/mac=$mac_addr" | head -n1)
+
+if [[ -z "$node" ]]; then
+  echo "ignition request for unknown mac $mac_addr" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+if ! $kubectl get "$node" -o jsonpath='{.metadata.annotations}' | jq -e '. | keys | any(. == "samcday.com/boot")' >/dev/null 2>&1; then
+  echo "ignition request for $node which is missing boot annotation" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+if ! $kubectl get "$node" -o jsonpath='{.metadata.annotations}' | jq -e '. | keys | any(. == "samcday.com/boot-profiles")' >/dev/null 2>&1; then
+  echo "ignition request for $node which is missing boot-profiles annotation" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+resp=$(curl --fail https://builds.coreos.fedoraproject.org/streams/stable.json | jq .architectures.x86_64.artifacts.metal.formats.pxe)
+
+kernel=$(jq -r .kernel.location    <<< "$resp")
+rootfs=$(jq -r .rootfs.location    <<< "$resp")
+initrd=$(jq -r .initramfs.location <<< "$resp")
+
+echo "content-type: application/json"
+echo
+cat <<HERE
+{
+  "kernel": "$kernel",
+  "initrd": ["$initrd"],
+  "cmdline": "coreos.live.rootfs_url={{ URL \"$rootfs\" }} coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url={{ URL \"/ignition/$mac_addr\" }}"
+}
+HERE

cluster/kube-system/pixiecore-api/generate-ignition.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -ueo pipefail
+
+export KUBECONFIG=/etc/kubernetes/admin.conf
+
+kubectl="chroot /host kubectl"
+kubeadm="chroot /host kubeadm"
+
+mac_addr=$(basename "$REQUEST_URI" | tr '[:upper:]' '[:lower:]' | tr ':' '-')
+node=$($kubectl get node -o name -l "samcday.com/mac=$mac_addr" | head -n1)
+
+if [[ -z "$node" ]]; then
+  echo "ignition request for unknown mac $mac_addr" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+bootprofiles="$($kubectl get "$node" -o jsonpath='{.metadata.annotations.samcday\.com/boot-profiles}')"
+
+if [[ -z "$bootprofiles" ]]; then
+  echo "ignition request for $node which is missing boot-profile annotations" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+if ! $kubectl get "$node" -o jsonpath='{.metadata.annotations}' | jq -e '. | keys | any(. == "samcday.com/boot")' >/dev/null 2>&1; then
+  echo "ignition request for $node which is missing boot annotation" >&2
+  echo "Status: 404"
+  echo
+  exit
+fi
+
+token=$($kubeadm token create)
+cahash=$(chroot /host bash -c "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt \
+       | openssl rsa -pubin -outform der 2>/dev/null \
+       | openssl dgst -sha256 -hex | sed 's/^.* //'")
+
+echo "content-type: application/json"
+echo
+
+if $kubectl get "$node" -o jsonpath='{.metadata.labels}' | jq -e '. | keys | any(. == "node-role.kubernetes.io/control-plane")' >/dev/null 2>&1; then
+  certkey=$($kubeadm certs certificate-key)
+  $kubeadm init phase upload-certs --upload-certs --certificate-key "$certkey" >&2
+  controlplane="
+            controlPlane:
+              certificateKey: \"$certkey\""
+fi
+
+IFS=","; for n in $bootprofiles; do
+  merge+="
+        - local: $n.ign"
+done
+
+butane -d /ignition --strict <<HERE
+variant: fcos
+version: 1.5.0
+ignition:
+  config:
+    merge: $merge
+storage:
+  files:
+    - path: /etc/kubeadm.conf.d/70-join.yaml
+      contents:
+        inline: |
+          ---
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: JoinConfiguration
+          nodeRegistration:
+              taints: []
+          $controlplane
+          discovery:
+              bootstrapToken:
+                  apiServerEndpoint: 10.0.1.9:6443
+                  token: "$token"
+                  caCertHashes: ["$cahash"]
+HERE

cluster/kube-system/pixiecore-api/nginx.conf

@@ -0,0 +1,16 @@
+server {
+    listen   80;
+
+    location /v1/boot/ {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass 127.0.0.1:9000;
+        fastcgi_param DOCUMENT_ROOT /tmp;
+        fastcgi_param SCRIPT_FILENAME /usr/local/bin/boot-request;
+    }
+    location /ignition {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_pass 127.0.0.1:9000;
+        fastcgi_param DOCUMENT_ROOT /tmp;
+        fastcgi_param SCRIPT_FILENAME /usr/local/bin/generate-ignition;
+    }
+}

cluster/kube-system/pixiecore.yaml

@@ -0,0 +1,88 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: pixiecore
+  namespace: kube-system
+spec:
+  chart:
+    spec:
+      chart: generic
+      sourceRef:
+        kind: HelmRepository
+        name: community-tooling-charts
+        namespace: flux-system
+      version: 7.5.6
+  interval: 1h
+  values:
+    # https://github.com/community-tooling/charts/blob/main/charts/generic/values.yaml
+    additionalContainers:
+      - name: api
+        image: alpine:3
+        command:
+          - ash
+          - -uexo
+          - pipefail
+          - -c
+          - |
+            apk add bash curl fcgiwrap jq nginx openssl # sops
+            apk add butane --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+            # 3.9.0 is quite new, is what ships the new (awesome) encrypted comment support
+            curl -sLo /usr/local/bin/sops https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64
+            chmod +x /usr/local/bin/sops
+
+            mkdir /ignition
+            for f in /butane/*.bu; do
+              name=$(basename $f)
+              cat $f
+              sops --input-type=yaml --output-type=yaml -d $f | butane --strict > /ignition/${name/.bu}.ign
+            done
+
+            install -m755 /scripts/boot-request.sh /usr/local/bin/boot-request
+            install -m755 /scripts/generate-ignition.sh /usr/local/bin/generate-ignition
+            install -m644 /scripts/nginx.conf /etc/nginx/http.d/default.conf
+
+            fcgiwrap -s 'tcp:0.0.0.0:9000' &
+            nginx -g "daemon off;"
+        env:
+          - name: SOPS_AGE_KEY
+            valueFrom:
+              secretKeyRef:
+                name: ignition-key
+                key: age.agekey
+        volumeMounts:
+          - name: host
+            mountPath: /host
+          - name: butane
+            mountPath: /butane
+          - name: scripts
+            mountPath: /scripts
+    additionalVolumes:
+      - name: host
+        hostPath:
+          path: /
+      - name: butane
+        configMap:
+          name: butane
+      - name: scripts
+        configMap:
+          name: pixiecore-api-scripts
+    additionalVolumeMounts:
+    args:
+      - api
+      - http://localhost:80
+      - --port
+      - '8080'
+    envValueFrom:
+    hostNetwork: true
+    image:
+      repository: pixiecore/pixiecore
+      tag: master
+    livenessProbe:
+      httpGet: ~
+    nodeSelector:
+      node-role.kubernetes.io/control-plane: ""
+    ports:
+    readinessProbe: ~
+    replicaCount: 1

cluster/secrets/kube-system/ignition-key.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: ignition-key
+    namespace: kube-system
+stringData:
+    age.agekey: ENC[AES256_GCM,data:iVU7zIZzhaGgpHKGqjcsN3WcbK0rta2Ro+tjwyVIdEbR6B562p+rLQk0OFOXg3E1bDYrF5BkErUdkP3i1V5vwBYd7aUUE+IAF55XSDhFZMgZDxa5ZL4jwoVg1pp8MZ8frQmSOx7Ist4gy97jORPxmlpul4h+B6xr5A9XaMl5+HwMKDPAnlpLOcKSaoY0QcKh2tp+Au45iok+3XYc4sxYJB15RoxYT3cwqQhQ+8g4OC02KXsTbTvM/iChgXo=,iv:IAVMU1BqEKF7v5Um5vZ0Ydr5xchDzyAwSyYuinlWVGg=,tag:TdiWojRm5DCdk/ZnpfLocQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNazdiNzNucnRqeGFYa1Z6
+            UFA5aVlHKy9NM3djZlRYNG5VY3o4bGtHTVFRCnhHaWpCWkJ0bmNsdXpRbmlEK0dB
+            elBUQzRMaXJXWm5menRhL3lkczZhVGMKLS0tIGhxbUR1SU04dklUQnlPV041ckhU
+            K0M0SXUycVRiVndWWk9sNUNjWFhXL1EK8PzTPc/jLpg2KRBXILm97RFLSuIrNkL6
+            MWvfzYvYrFPZmOD8Yat8cjw7mko3xSUDjK4WUGQGXSjQqEqnFcA9VA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-07-01T17:19:57Z"
+    mac: ENC[AES256_GCM,data:f65G/w6K42atTABrS8NSYVGmA2MRs+5HksTjQnps2Ul8/PxoqMqRF+lMPsUjn28pYW+tdm1palMJVpLVBdf1/z2+qBp0iVpd8kZg4jZK3TSd+1Fj7K+SpCRLtdR/H4ac8I8xETGa4YBulLSHD9cbJXyQCUz0kFsBYYsNNoBjwo0=,iv:81g4AjS1tnaStVcPgnwd8KTnpDiv2OuxJaIZKNjzy7Y=,tag:Gq9rKNIuWlzhcgCq9BOfMA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.0