From 8aa6041b5a719e4af801000da63b8372ef5bafe6 Mon Sep 17 00:00:00 2001 From: Jason Gilfoil Date: Sun, 25 Feb 2024 03:17:56 +0000 Subject: [PATCH 1/3] deploy rook-ceph --- .../apps/rook-ceph/app/helmrelease.yaml | 42 ++++++ .../apps/rook-ceph/app/kustomization.yaml | 7 + ...k-ceph-dashboard-password.secret.sops.yaml | 27 ++++ .../apps/rook-ceph/cluster/helmrelease.yaml | 137 ++++++++++++++++++ .../apps/rook-ceph/cluster/kustomization.yaml | 7 + .../apps/rook-ceph/cluster/rgw-external.yaml | 53 +++++++ kubernetes/apps/rook-ceph/ks.yaml | 42 ++++++ kubernetes/apps/rook-ceph/kustomization.yaml | 9 ++ kubernetes/apps/rook-ceph/namespace.yaml | 7 + 9 files changed, 331 insertions(+) create mode 100644 kubernetes/apps/rook-ceph/app/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/app/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml create mode 100644 kubernetes/apps/rook-ceph/cluster/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/cluster/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/cluster/rgw-external.yaml create mode 100644 kubernetes/apps/rook-ceph/ks.yaml create mode 100644 kubernetes/apps/rook-ceph/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/namespace.yaml diff --git a/kubernetes/apps/rook-ceph/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/app/helmrelease.yaml new file mode 100644 index 00000000..6287698e --- /dev/null +++ b/kubernetes/apps/rook-ceph/app/helmrelease.yaml @@ -0,0 +1,42 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: rook-ceph-operator +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph + version: v1.13.3 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: snapshot-controller + namespace: storage + values: + csi: + cephFSKernelMountOptions: ms_mode=prefer-crc + enableLiveness: true + serviceMonitor: + enabled: true + monitoring: + enabled: true + resources: + requests: + memory: 128Mi # unchangable + cpu: 100m # unchangable + limits: {} diff --git a/kubernetes/apps/rook-ceph/app/kustomization.yaml b/kubernetes/apps/rook-ceph/app/kustomization.yaml new file mode 100644 index 00000000..fb2f8c12 --- /dev/null +++ b/kubernetes/apps/rook-ceph/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./rook-ceph-dashboard-password.secret.sops.yaml + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml b/kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml new file mode 100644 index 00000000..d1f067cf --- /dev/null +++ b/kubernetes/apps/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml @@ -0,0 +1,27 @@ +# yamllint disable +apiVersion: v1 +kind: Secret +metadata: + name: rook-ceph-dashboard-password +stringData: + password: ENC[AES256_GCM,data:QXg+80UQp4OEg5lXyk6/cA==,iv:9N8bvoRJ8ANhaG8HQId9+sLcoL3r97FEToo/FZ6evIo=,tag:8xGrcYmfpOwMqstFdC+/Eg==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age12rzrdtn8xhd89y23qw4kymxftuylqn5cm522jcn327atent4a40swjcgmj + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIWjliYXU5emg1TnI5SEcv + bjMxTlVVeURpeWV1Qys4V0w1RnNyMzFEL1JVClhiRXpVUjdjQTgzdkNqZ0V1MXkw + K3hQWm9Bb0dKWGtQTXFrNm5wSytIV1EKLS0tIC9IbXpoVFk4NWhrdXNiRFkvYm90 + REJGSU5ONXJHWWNXbGE0Q21KdFpsbmcKAibEc4C50OtZiQkGEHEF+YJ9uzHmreFJ + VDsXzxStCsgBUSfG7QxRg2RSlHxsBhu/n27iB+Mcz0P7on5zlCBPXA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-02-25T02:55:34Z" + mac: ENC[AES256_GCM,data:j+bE8tT8Kv7yJBE2zwxr8KU8V4gIf6M2Gh3ZfgAReikDGkJZfvXxueVW51h18GnbMUZ+UUiLXj1cPQHsFIkIUwyK7eMZ5Yd8Gtzg+9GALwoA2MK8n98vBmRb6RjO93UyJuMpTH1AgsEp/5Sqk7UVKxMf2SXcaJihjmaLopGkkmg=,iv:qQg2E49tuE2N0o3PC6J0I6xCoq7onfPDl46yclm7MvU=,tag:Uotdx2lqMxQ4rR2S4fS15w==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml new file mode 100644 index 00000000..b747b5cf --- /dev/null +++ b/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml @@ -0,0 +1,137 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: rook-ceph-cluster +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph-cluster + version: v1.13.3 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: rook-ceph-operator + namespace: rook-ceph + - name: snapshot-controller + namespace: storage + values: + monitoring: + enabled: true + createPrometheusRules: true + ingress: + dashboard: + ingressClassName: internal + host: + name: &host rook.${SECRET_DOMAIN} + path: / + tls: + - hosts: + - *host + toolbox: + enabled: true + configOverride: | + [global] + bdev_enable_discard = true + bdev_async_discard = true + osd_class_update_on_start = false + cephClusterSpec: + network: + provider: host + connections: + requireMsgr2: true + crashCollector: + disable: false + dashboard: + enabled: true + urlPrefix: / + ssl: false + storage: + useAllNodes: false + useAllDevices: false + config: + osdsPerDevice: "1" + nodes: + - name: odroid-01 + devices: + - name: /dev/sda + - name: odroid-02 + devices: + - name: /dev/sda + - name: odroid-03 + devices: + - name: /dev/sda + placement: + mgr: &placement + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + mon: *placement + resources: + mgr: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 2000m + memory: 2Gi + mon: + requests: + cpu: 1000m + memory: 1Gi + limits: + cpu: 4000m + memory: 4Gi + osd: + requests: + cpu: 1000m + memory: 4Gi + limits: + cpu: 4000m + memory: 8Gi + cephBlockPools: + - name: ceph-blockpool + spec: + failureDomain: host + replicated: + size: 3 + storageClass: + enabled: true + name: ceph-block + isDefault: true + reclaimPolicy: Delete + allowVolumeExpansion: true + parameters: + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 + cephBlockPoolsVolumeSnapshotClass: + enabled: true + name: csi-ceph-blockpool + isDefault: false + deletionPolicy: Delete + cephFileSystems: [] + cephFileSystemVolumeSnapshotClass: [] \ No newline at end of file diff --git a/kubernetes/apps/rook-ceph/cluster/kustomization.yaml b/kubernetes/apps/rook-ceph/cluster/kustomization.yaml new file mode 100644 index 00000000..03119d35 --- /dev/null +++ b/kubernetes/apps/rook-ceph/cluster/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + # - ./rgw-external.yaml diff --git a/kubernetes/apps/rook-ceph/cluster/rgw-external.yaml b/kubernetes/apps/rook-ceph/cluster/rgw-external.yaml new file mode 100644 index 00000000..39fd5acf --- /dev/null +++ b/kubernetes/apps/rook-ceph/cluster/rgw-external.yaml @@ -0,0 +1,53 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/ceph.rook.io/cephobjectstoreuser_v1.json +apiVersion: ceph.rook.io/v1 +kind: CephObjectStoreUser +metadata: + name: cluster-admin +spec: + # Ref: https://rook.io/docs/rook/v1.13/Storage-Configuration/Object-Storage-RGW/object-storage/ + store: ceph-objectstore + displayName: Cluster Admin +--- +apiVersion: v1 +kind: Service +metadata: + name: rook-ceph-rgw-ceph-objectstore-external + namespace: rook-ceph + labels: + app: rook-ceph-rgw + rook_cluster: rook-ceph + rook_object_store: ceph-objectstore +spec: + type: NodePort + selector: + app: rook-ceph-rgw + rook_cluster: rook-ceph + rook_object_store: ceph-objectstore + ports: + - name: rgw + port: 80 + protocol: TCP + targetPort: 80 + sessionAffinity: None +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rook-ceph-rgw +spec: + ingressClassName: internal + rules: + - host: &host rook-ceph-rgw.${SECRET_DOMAIN} + http: + paths: + - backend: + service: + name: rook-ceph-rgw-ceph-objectstore-external + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - *host diff --git a/kubernetes/apps/rook-ceph/ks.yaml b/kubernetes/apps/rook-ceph/ks.yaml new file mode 100644 index 00000000..295ec9eb --- /dev/null +++ b/kubernetes/apps/rook-ceph/ks.yaml @@ -0,0 +1,42 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app rook-ceph + namespace: flux-system +spec: + targetNamespace: rook-ceph + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/rook-ceph/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 5m30s + timeout: 5m +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app rook-ceph-cluster + namespace: flux-system +spec: + targetNamespace: rook-ceph + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/rook-ceph/cluster + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 5m30s + timeout: 15m diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/kubernetes/apps/rook-ceph/kustomization.yaml new file mode 100644 index 00000000..14fac376 --- /dev/null +++ b/kubernetes/apps/rook-ceph/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Pre Flux-Kustomizations + - ./namespace.yaml + # Flux-Kustomizations + - ./ks.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/kubernetes/apps/rook-ceph/namespace.yaml new file mode 100644 index 00000000..4f4d74a8 --- /dev/null +++ b/kubernetes/apps/rook-ceph/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph + labels: + kustomize.toolkit.fluxcd.io/prune: disabled From 33bb91d78afa50cd9cbba4be715e9f0dbaa1ef0f Mon Sep 17 00:00:00 2001 From: Jason Gilfoil Date: Sun, 25 Feb 2024 03:27:39 +0000 Subject: [PATCH 2/3] clean up flux respository sources --- kubernetes/flux/repositories/helm/intel.yaml | 10 ++++++++++ kubernetes/flux/repositories/helm/kustomization.yaml | 6 +----- .../helm/{weave-gitops.yaml => metallb.yaml} | 8 ++++---- ...csi-driver-smb.yaml => node-feature-discovery.yaml} | 5 +++-- .../helm/{longhorn.yaml => rook-ceph.yaml} | 5 +++-- .../{csi-driver-nfs.yaml => weaveworks-kured.yaml} | 5 +++-- 6 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 kubernetes/flux/repositories/helm/intel.yaml rename kubernetes/flux/repositories/helm/{weave-gitops.yaml => metallb.yaml} (56%) rename kubernetes/flux/repositories/helm/{csi-driver-smb.yaml => node-feature-discovery.yaml} (52%) rename kubernetes/flux/repositories/helm/{longhorn.yaml => rook-ceph.yaml} (64%) rename kubernetes/flux/repositories/helm/{csi-driver-nfs.yaml => weaveworks-kured.yaml} (53%) diff --git a/kubernetes/flux/repositories/helm/intel.yaml b/kubernetes/flux/repositories/helm/intel.yaml new file mode 100644 index 00000000..3fedf7c6 --- /dev/null +++ b/kubernetes/flux/repositories/helm/intel.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: intel + namespace: flux-system +spec: + interval: 2h + url: https://intel.github.io/helm-charts diff --git a/kubernetes/flux/repositories/helm/kustomization.yaml b/kubernetes/flux/repositories/helm/kustomization.yaml index 29df8aaa..c84ffb15 100644 --- a/kubernetes/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/flux/repositories/helm/kustomization.yaml @@ -7,21 +7,17 @@ resources: - ./bjw-s.yaml - ./cilium.yaml - ./coredns.yaml - - ./csi-driver-nfs.yaml - - ./csi-driver-smb.yaml - ./descheduler.yaml - ./external-dns.yaml - ./grafana.yaml - ./ingress-nginx.yaml - ./jetstack.yaml - - ./k8s-gateway.yaml - ./kubernetes-dashboard.yaml - - ./longhorn.yaml - ./metrics-server.yaml - ./openebs.yaml - ./piraeus.yaml - ./postfinance.yaml - ./prometheus-community.yaml + - ./rook-ceph.yaml - ./stakater.yaml - - ./weave-gitops.yaml - ./xenitab.yaml diff --git a/kubernetes/flux/repositories/helm/weave-gitops.yaml b/kubernetes/flux/repositories/helm/metallb.yaml similarity index 56% rename from kubernetes/flux/repositories/helm/weave-gitops.yaml rename to kubernetes/flux/repositories/helm/metallb.yaml index f325c18b..fa13bbe9 100644 --- a/kubernetes/flux/repositories/helm/weave-gitops.yaml +++ b/kubernetes/flux/repositories/helm/metallb.yaml @@ -2,9 +2,9 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: weave-gitops + name: metallb namespace: flux-system spec: - type: oci - interval: 5m - url: oci://ghcr.io/weaveworks/charts + interval: 1h + url: https://metallb.github.io/metallb + timeout: 3m diff --git a/kubernetes/flux/repositories/helm/csi-driver-smb.yaml b/kubernetes/flux/repositories/helm/node-feature-discovery.yaml similarity index 52% rename from kubernetes/flux/repositories/helm/csi-driver-smb.yaml rename to kubernetes/flux/repositories/helm/node-feature-discovery.yaml index a35bd825..9f8f522b 100644 --- a/kubernetes/flux/repositories/helm/csi-driver-smb.yaml +++ b/kubernetes/flux/repositories/helm/node-feature-discovery.yaml @@ -2,8 +2,9 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: csi-driver-smb + name: node-feature-discovery namespace: flux-system spec: interval: 1h - url: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts + url: https://kubernetes-sigs.github.io/node-feature-discovery/charts + timeout: 3m diff --git a/kubernetes/flux/repositories/helm/longhorn.yaml b/kubernetes/flux/repositories/helm/rook-ceph.yaml similarity index 64% rename from kubernetes/flux/repositories/helm/longhorn.yaml rename to kubernetes/flux/repositories/helm/rook-ceph.yaml index bc41510b..23c25530 100644 --- a/kubernetes/flux/repositories/helm/longhorn.yaml +++ b/kubernetes/flux/repositories/helm/rook-ceph.yaml @@ -2,8 +2,9 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: longhorn + name: rook-ceph namespace: flux-system spec: interval: 1h - url: https://charts.longhorn.io + url: https://charts.rook.io/release + timeout: 3m diff --git a/kubernetes/flux/repositories/helm/csi-driver-nfs.yaml b/kubernetes/flux/repositories/helm/weaveworks-kured.yaml similarity index 53% rename from kubernetes/flux/repositories/helm/csi-driver-nfs.yaml rename to kubernetes/flux/repositories/helm/weaveworks-kured.yaml index b48140d7..060d04d4 100644 --- a/kubernetes/flux/repositories/helm/csi-driver-nfs.yaml +++ b/kubernetes/flux/repositories/helm/weaveworks-kured.yaml @@ -2,8 +2,9 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: csi-driver-nfs + name: weaveworks-kured-charts namespace: flux-system spec: interval: 1h - url: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts + url: https://kubereboot.github.io/charts + timeout: 3m From a03810559a9a78b9381b8278aa4c242e0235940c Mon Sep 17 00:00:00 2001 From: Jason Gilfoil Date: Sun, 25 Feb 2024 03:36:32 +0000 Subject: [PATCH 3/3] remove empty values in rook-ceph-cluster chart was getting errors from flux diff --- kubernetes/apps/rook-ceph/cluster/helmrelease.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml index b747b5cf..6246a8b8 100644 --- a/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/apps/rook-ceph/cluster/helmrelease.yaml @@ -133,5 +133,3 @@ spec: name: csi-ceph-blockpool isDefault: false deletionPolicy: Delete - cephFileSystems: [] - cephFileSystemVolumeSnapshotClass: [] \ No newline at end of file