Merge pull request #5 from jgilfoil/feature/rook-ceph

deploy rook-ceph
jgilfoil · Feb 25, 2024 · 388273c · 388273c
2 parents baf204e + a038105
commit 388273c
Show file tree

Hide file tree

Showing 15 changed files with 353 additions and 15 deletions.
diff --git a/kubernetes/apps/rook-ceph/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/app/helmrelease.yaml
@@ -0,0 +1,42 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: rook-ceph-operator
+spec:
+  interval: 30m
+  timeout: 15m
+  chart:
+    spec:
+      chart: rook-ceph
+      version: v1.13.3
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: snapshot-controller
+      namespace: storage
+  values:
+    csi:
+      cephFSKernelMountOptions: ms_mode=prefer-crc
+      enableLiveness: true
+      serviceMonitor:
+        enabled: true
+    monitoring:
+      enabled: true
+    resources:
+      requests:
+        memory: 128Mi # unchangable
+        cpu: 100m # unchangable
+      limits: {}
diff --git a/kubernetes/apps/rook-ceph/app/kustomization.yaml b/kubernetes/apps/rook-ceph/app/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./rook-ceph-dashboard-password.secret.sops.yaml
+  - ./helmrelease.yaml
diff --git a/kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml b/kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml
@@ -0,0 +1,27 @@
+# yamllint disable
+apiVersion: v1
+kind: Secret
+metadata:
+    name: rook-ceph-dashboard-password
+stringData:
+    password: ENC[AES256_GCM,data:QXg+80UQp4OEg5lXyk6/cA==,iv:9N8bvoRJ8ANhaG8HQId9+sLcoL3r97FEToo/FZ6evIo=,tag:8xGrcYmfpOwMqstFdC+/Eg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age12rzrdtn8xhd89y23qw4kymxftuylqn5cm522jcn327atent4a40swjcgmj
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIWjliYXU5emg1TnI5SEcv
+            bjMxTlVVeURpeWV1Qys4V0w1RnNyMzFEL1JVClhiRXpVUjdjQTgzdkNqZ0V1MXkw
+            K3hQWm9Bb0dKWGtQTXFrNm5wSytIV1EKLS0tIC9IbXpoVFk4NWhrdXNiRFkvYm90
+            REJGSU5ONXJHWWNXbGE0Q21KdFpsbmcKAibEc4C50OtZiQkGEHEF+YJ9uzHmreFJ
+            VDsXzxStCsgBUSfG7QxRg2RSlHxsBhu/n27iB+Mcz0P7on5zlCBPXA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-02-25T02:55:34Z"
+    mac: ENC[AES256_GCM,data:j+bE8tT8Kv7yJBE2zwxr8KU8V4gIf6M2Gh3ZfgAReikDGkJZfvXxueVW51h18GnbMUZ+UUiLXj1cPQHsFIkIUwyK7eMZ5Yd8Gtzg+9GALwoA2MK8n98vBmRb6RjO93UyJuMpTH1AgsEp/5Sqk7UVKxMf2SXcaJihjmaLopGkkmg=,iv:qQg2E49tuE2N0o3PC6J0I6xCoq7onfPDl46yclm7MvU=,tag:Uotdx2lqMxQ4rR2S4fS15w==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
diff --git a/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml
@@ -0,0 +1,135 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: rook-ceph-cluster
+spec:
+  interval: 30m
+  timeout: 15m
+  chart:
+    spec:
+      chart: rook-ceph-cluster
+      version: v1.13.3
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: rook-ceph-operator
+      namespace: rook-ceph
+    - name: snapshot-controller
+      namespace: storage
+  values:
+    monitoring:
+      enabled: true
+      createPrometheusRules: true
+    ingress:
+      dashboard:
+        ingressClassName: internal
+        host:
+          name: &host rook.${SECRET_DOMAIN}
+          path: /
+        tls:
+          - hosts:
+              - *host
+    toolbox:
+      enabled: true
+    configOverride: |
+      [global]
+      bdev_enable_discard = true
+      bdev_async_discard = true
+      osd_class_update_on_start = false
+    cephClusterSpec:
+      network:
+        provider: host
+        connections:
+          requireMsgr2: true
+      crashCollector:
+        disable: false
+      dashboard:
+        enabled: true
+        urlPrefix: /
+        ssl: false
+      storage:
+        useAllNodes: false
+        useAllDevices: false
+        config:
+          osdsPerDevice: "1"
+        nodes:
+          - name: odroid-01
+            devices:
+              - name: /dev/sda
+          - name: odroid-02
+            devices:
+              - name: /dev/sda
+          - name: odroid-03
+            devices:
+              - name: /dev/sda
+      placement:
+        mgr: &placement
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+                - matchExpressions:
+                    - key: node-role.kubernetes.io/control-plane
+                      operator: Exists
+        mon: *placement
+      resources:
+        mgr:
+          requests:
+            cpu: 500m
+            memory: 512Mi
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+        mon:
+          requests:
+            cpu: 1000m
+            memory: 1Gi
+          limits:
+            cpu: 4000m
+            memory: 4Gi
+        osd:
+          requests:
+            cpu: 1000m
+            memory: 4Gi
+          limits:
+            cpu: 4000m
+            memory: 8Gi
+    cephBlockPools:
+      - name: ceph-blockpool
+        spec:
+          failureDomain: host
+          replicated:
+            size: 3
+        storageClass:
+          enabled: true
+          name: ceph-block
+          isDefault: true
+          reclaimPolicy: Delete
+          allowVolumeExpansion: true
+          parameters:
+            imageFormat: "2"
+            imageFeatures: layering
+            csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+            csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+            csi.storage.k8s.io/fstype: ext4
+    cephBlockPoolsVolumeSnapshotClass:
+      enabled: true
+      name: csi-ceph-blockpool
+      isDefault: false
+      deletionPolicy: Delete
diff --git a/kubernetes/apps/rook-ceph/cluster/kustomization.yaml b/kubernetes/apps/rook-ceph/cluster/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  # - ./rgw-external.yaml
diff --git a/kubernetes/apps/rook-ceph/cluster/rgw-external.yaml b/kubernetes/apps/rook-ceph/cluster/rgw-external.yaml
@@ -0,0 +1,53 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/ceph.rook.io/cephobjectstoreuser_v1.json
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: cluster-admin
+spec:
+  # Ref: https://rook.io/docs/rook/v1.13/Storage-Configuration/Object-Storage-RGW/object-storage/
+  store: ceph-objectstore
+  displayName: Cluster Admin
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rook-ceph-rgw-ceph-objectstore-external
+  namespace: rook-ceph
+  labels:
+    app: rook-ceph-rgw
+    rook_cluster: rook-ceph
+    rook_object_store: ceph-objectstore
+spec:
+  type: NodePort
+  selector:
+    app: rook-ceph-rgw
+    rook_cluster: rook-ceph
+    rook_object_store: ceph-objectstore
+  ports:
+    - name: rgw
+      port: 80
+      protocol: TCP
+      targetPort: 80
+  sessionAffinity: None
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rook-ceph-rgw
+spec:
+  ingressClassName: internal
+  rules:
+    - host: &host rook-ceph-rgw.${SECRET_DOMAIN}
+      http:
+        paths:
+          - backend:
+              service:
+                name: rook-ceph-rgw-ceph-objectstore-external
+                port:
+                  number: 80
+            path: /
+            pathType: Prefix
+  tls:
+    - hosts:
+        - *host
diff --git a/kubernetes/apps/rook-ceph/ks.yaml b/kubernetes/apps/rook-ceph/ks.yaml
@@ -0,0 +1,42 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app rook-ceph
+  namespace: flux-system
+spec:
+  targetNamespace: rook-ceph
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/rook-ceph/app
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 5m30s
+  timeout: 5m
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app rook-ceph-cluster
+  namespace: flux-system
+spec:
+  targetNamespace: rook-ceph
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/rook-ceph/cluster
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 5m30s
+  timeout: 15m
diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/kubernetes/apps/rook-ceph/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # Pre Flux-Kustomizations
+  - ./namespace.yaml
+  # Flux-Kustomizations
+  - ./ks.yaml
diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/kubernetes/apps/rook-ceph/namespace.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rook-ceph
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
diff --git a/kubernetes/flux/repositories/helm/intel.yaml b/kubernetes/flux/repositories/helm/intel.yaml
@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: intel
+  namespace: flux-system
+spec:
+  interval: 2h
+  url: https://intel.github.io/helm-charts
diff --git a/kubernetes/flux/repositories/helm/kustomization.yaml b/kubernetes/flux/repositories/helm/kustomization.yaml
@@ -7,21 +7,17 @@ resources:
   - ./bjw-s.yaml
   - ./cilium.yaml
   - ./coredns.yaml
-  - ./csi-driver-nfs.yaml
-  - ./csi-driver-smb.yaml
   - ./descheduler.yaml
   - ./external-dns.yaml
   - ./grafana.yaml
   - ./ingress-nginx.yaml
   - ./jetstack.yaml
-  - ./k8s-gateway.yaml
   - ./kubernetes-dashboard.yaml
-  - ./longhorn.yaml
   - ./metrics-server.yaml
   - ./openebs.yaml
   - ./piraeus.yaml
   - ./postfinance.yaml
   - ./prometheus-community.yaml
+  - ./rook-ceph.yaml
   - ./stakater.yaml
-  - ./weave-gitops.yaml
   - ./xenitab.yaml
diff --git a/.../flux/repositories/helm/weave-gitops.yaml → ...netes/flux/repositories/helm/metallb.yaml b/.../flux/repositories/helm/weave-gitops.yaml → ...netes/flux/repositories/helm/metallb.yaml
@@ -2,9 +2,9 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
-  name: weave-gitops
+  name: metallb
   namespace: flux-system
 spec:
-  type: oci
-  interval: 5m
-  url: oci://ghcr.io/weaveworks/charts
+  interval: 1h
+  url: https://metallb.github.io/metallb
+  timeout: 3m
diff --git a/...lux/repositories/helm/csi-driver-smb.yaml → ...sitories/helm/node-feature-discovery.yaml b/...lux/repositories/helm/csi-driver-smb.yaml → ...sitories/helm/node-feature-discovery.yaml
@@ -2,8 +2,9 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
-  name: csi-driver-smb
+  name: node-feature-discovery
   namespace: flux-system
 spec:
   interval: 1h
-  url: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts
+  url: https://kubernetes-sigs.github.io/node-feature-discovery/charts
+  timeout: 3m