Poc deploy operator

Makefile

@@ -1,8 +1,9 @@
 UNAME_S := $(shell uname -s)
+NC := $(shell tput sgr0) # No Color
 ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
-  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.21.0/kind-linux-amd64
+  K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -12,16 +13,20 @@ endif
 ifeq ($(UNAME_S),Darwin)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
-  KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.21.0/kind-darwin-amd64
+  K3D_BIN ?=  https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
   KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
   YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
   JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
   OPM_TAR ?= https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.8/opm-mac-4.8.57.tar.gz
   OPM_BIN ?= darwin-amd64-opm
 endif
 
-KIND_CLUSTER ?= chart-testing
-REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
+K3D_CLUSTER ?= chart-testing
+REGISTRY ?= gcr.io
+REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
+DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
+LOCAL_REGISTRY ?= "localhost:5000"
+CLUSTER_SIZE ?= 1
 
 export BUNDLE_IMAGE ?= cockroach-operator-bundle
 export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
@@ -56,7 +61,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
 build/self-signer: bin/yq ## build the self-signer image
 	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Release
 
@@ -66,33 +71,61 @@ release: ## publish the build artifacts to S3
 build-and-push/self-signer: bin/yq ## push the self-signer image
 	@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Dev
 dev/clean: ## remove built artifacts
 	@rm -r build/artifacts/
 
+## Setup/teardown registries for easier local dev
+dev/registries/up: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
+		echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/down: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
+		echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up
+
+dev/push/local: dev/registries/up
+	@echo "$(CYAN)Pushing image to local registry...$(NC)"
+	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
+          	--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
+          	-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+
 ##@ Test
+test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing
+
+test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing
+
+test/cluster/up: bin/k3d
+	@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)
 
-test/cluster: bin/kind ## start a local kind cluster for testing
-	@bin/kind get clusters -q | grep $(KIND_CLUSTER) || bin/kind create cluster --name $(KIND_CLUSTER)
+test/cluster/down: bin/k3d
+	./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"
 
 test/e2e/%: PKG=$*
-test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-kind ## run e2e tests for package (e.g. install or rotate)
+test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/...
 
 test/lint: bin/helm ## lint the helm chart
 	@build/lint.sh && bin/helm lint cockroachdb
 
 IMAGE_LIST = cockroachdb/cockroach:v23.2.0 quay.io/jetstack/cert-manager-cainjector:v1.11.0 quay.io/jetstack/cert-manager-webhook:v1.11.0 quay.io/jetstack/cert-manager-controller:v1.11.0 quay.io/jetstack/cert-manager-ctl:v1.11.0
-test/publish-images-to-kind: bin/yq test/cluster ## publish signer and cockroach image to local kind registry
+test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach image to local k3d registry
 	for i in $(IMAGE_LIST); do \
 		docker pull $$i; \
-		bin/kind load docker-image $$i --name $(KIND_CLUSTER); \
+		bin/k3d image import $$i -c $(K3D_CLUSTER); \
 	done
-	@bin/kind load docker-image \
-		${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
-		--name $(KIND_CLUSTER)
+	docker pull ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
+	bin/k3d image import \
+		${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
+		-c $(K3D_CLUSTER)
 
 test/template: bin/cockroach bin/helm ## Run template tests
 	@PATH="$(PWD)/bin:${PATH}" go test -v ./tests/template/...
@@ -101,7 +134,7 @@ test/units: bin/cockroach ## Run unit tests in ./pkg/...
 	@PATH="$(PWD)/bin:${PATH}" go test -v ./pkg/...
 
 ##@ Binaries
-bin: bin/cockroach bin/helm bin/kind bin/kubectl bin/yq ## install all binaries
+bin: bin/cockroach bin/helm bin/k3d bin/kubectl bin/yq ## install all binaries
 
 bin/cockroach: ## install cockroach
 	@mkdir -p bin
@@ -113,10 +146,10 @@ bin/helm: ## install helm
 	@curl -L $(HELM_BIN) | tar -xzf - -C bin/ --strip-components 1
 	@rm -f bin/README.md bin/LICENSE
 
-bin/kind: ## install kind
+bin/k3d: ## install k3d
 	@mkdir -p bin
-	@curl -Lo bin/kind $(KIND_BIN)	
-	@chmod +x bin/kind
+	@curl -Lo bin/k3d $(K3D_BIN)	
+	@chmod +x bin/k3d
 
 bin/kubectl: ## install kubectl
 	@mkdir -p bin

cmd/self-signer/generate.go

@@ -37,10 +37,12 @@ var (
 	caExpiry, nodeExpiry, clientExpiry       string
 	caSecret                                 string
 	clientOnly                               bool
+	operatorManaged                          bool
 )
 
 func init() {
 	generateCmd.Flags().BoolVar(&clientOnly, "client-only", false, "generate certificates for custom user")
+	generateCmd.Flags().BoolVar(&operatorManaged, "operator-managed", false, "generate certificates for operator managed cluster")
 	rootCmd.AddCommand(generateCmd)
 }
 
@@ -52,6 +54,7 @@ func generate(cmd *cobra.Command, args []string) {
 	}
 
 	genCert.CaSecret = caSecret
+	genCert.OperatorManaged = operatorManaged
 
 	namespace, exists := os.LookupEnv("NAMESPACE")
 	if !exists {

cockroachdb/Chart.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 name: cockroachdb
 home: https://www.cockroachlabs.com
-version: 14.0.5
+version: 14.0.6
 appVersion: 24.2.4
 description: CockroachDB is a scalable, survivable, strongly-consistent SQL database.
 icon: https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png

cockroachdb/templates/_helpers.tpl

@@ -289,3 +289,23 @@ Validate that if user enabled tls, then either self-signed certificates or certi
 {{- end }}
 {{- end }}
 {{- end }}
+
+
+{{/* Common labels that are applied to all managed objects. */}}
+{{- define "cluster.labels" -}}
+helm.sh/chart: {{ include "cockroachdb.chart" . }}
+{{ include "cluster.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+  Selector labels defines the set of labels that can be used as selectors for 
+  crdb nodes.
+*/}}
+{{- define "cluster.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "cockroachdb.clusterfullname" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

cockroachdb/templates/crdb.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.operator.enabled }}
+---
+apiVersion: crdb.cockroachlabs.com/v1alpha1
+kind: CrdbCluster
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cluster.labels" . | nindent 4 }}
+  {{- with .Values.statefulset.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.operator.clusterSettings }}
+  clusterSettings: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.operator.regions }}
+  regions: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  features:
+    - reconcile
+    - reconcile-beta
+  template:
+    spec:
+      image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
+      certificates:
+        externalCertificates:
+          clientCaConfigMapName: {{ .Values.operator.certificates.externalCertificates.clientCaConfigMapName | default (printf "%s-client-ca" $.Release.Name) }}
+          nodeCaConfigMapName: {{ .Values.operator.certificates.externalCertificates.nodeCaConfigMapName | default (printf "%s-node-ca" $.Release.Name) }}
+          httpSecretName: {{ .Values.operator.certificates.externalCertificates.httpSecretName | default (printf "%s-ui-certs" $.Release.Name) }} 
+          nodeClientSecretName: {{ .Values.operator.certificates.externalCertificates.nodeClientSecretName | default (printf "%s-node-client-certs" $.Release.Name) }} 
+          nodeSecretName: {{ .Values.operator.certificates.externalCertificates.nodeSecretName | default (printf "%s-node-certs" $.Release.Name) }} 
+          rootSqlClientSecretName: {{ .Values.operator.certificates.externalCertificates.rootSqlClientSecretName | default (printf "%s-client-certs" $.Release.Name) }}
+    {{- with .Values.operator.resources }}
+      resourceRequirements: {{- toYaml . | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ default .Release.Name .Values.operator.rbac.serviceAccountName }}
+    {{- if .Values.operator.loggingConf }}
+      loggingConfigMapName: {{ .Release.Name }}-logging
+    {{- end }}
+  # All properties below are solely to pass validation. They aren't used by the
+  # betaclusterctrl controller so the values don't matter so long as they're
+  # valid.
+  dataStore: {}
+{{- end }}

cockroachdb/templates/job-certSelfSigner.yaml

@@ -66,6 +66,9 @@ spec:
             - --client-expiry={{ .Values.tls.certs.selfSigner.clientCertExpiryWindow }}
             - --node-duration={{ .Values.tls.certs.selfSigner.nodeCertDuration }}
             - --node-expiry={{ .Values.tls.certs.selfSigner.nodeCertExpiryWindow }}
+            {{- if .Values.operator.enabled }}
+            - --operator-managed=true
+            {{- end}}
           env:
           - name: STATEFULSET_NAME
             value: {{ template "cockroachdb.fullname" . }}

cockroachdb/templates/job.init.yaml

@@ -1,6 +1,6 @@
 {{ $isClusterInitEnabled := and (eq (len .Values.conf.join) 0) (not (index .Values.conf `single-node`)) }}
 {{ $isDatabaseProvisioningEnabled := .Values.init.provisioning.enabled }}
-{{- if or $isClusterInitEnabled $isDatabaseProvisioningEnabled }}
+{{- if and (or $isClusterInitEnabled $isDatabaseProvisioningEnabled) (not .Values.operator.enabled) }}
   {{ template "cockroachdb.tlsValidation" . }}
 kind: Job
 apiVersion: batch/v1

cockroachdb/templates/poddisruptionbudget.yaml

@@ -1,3 +1,4 @@
+{{- if (not .Values.operator.enabled) }}
 kind: PodDisruptionBudget
 {{- if or (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version) }}
 apiVersion: policy/v1
@@ -24,3 +25,4 @@ spec:
       {{- toYaml . | nindent 6 }}
     {{- end }}
   maxUnavailable: {{ .Values.statefulset.budget.maxUnavailable | int64 }}
+{{- end }}

cockroachdb/templates/role-certSelfSigner.yaml

@@ -30,4 +30,7 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["delete", "get"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create", "get", "update", "delete"]
 {{- end }}

cockroachdb/templates/role.yaml

@@ -20,4 +20,25 @@ rules:
     {{- else }}
     verbs: ["create", "get"]
     {{- end }}
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    helm.sh/chart: {{ template "cockroachdb.chart" . }}
+    app.kubernetes.io/name: {{ template "cockroachdb.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
 {{- end }}

cockroachdb/templates/rolebinding.yaml

@@ -20,4 +20,26 @@ subjects:
   - kind: ServiceAccount
     name: {{ template "cockroachdb.serviceAccount.name" . }}
     namespace: {{ .Release.Namespace | quote }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    helm.sh/chart: {{ template "cockroachdb.chart" . }}
+    app.kubernetes.io/name: {{ template "cockroachdb.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "cockroachdb.serviceAccount.name" . }}
+    namespace: {{ .Release.Namespace | quote }}
 {{- end }}

cockroachdb/templates/statefulset.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.operator.enabled }}
 kind: StatefulSet
 apiVersion: {{ template "cockroachdb.statefulset.apiVersion" . }}
 metadata:
@@ -433,3 +434,4 @@ spec:
           requests:
             storage: {{ .Values.storage.persistentVolume.size | quote }}
 {{- end }}
+{{- end }}