From 3d50513b417261e22821fd6644b7145c86f8e213 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Mon, 11 Nov 2024 00:38:32 -0700
Subject: [PATCH 01/41] feat: Add staringOrdinal

---
 api/v1/cosmosfullnode_types.go                |  4 +++
 api/v1/zz_generated.deepcopy.go               |  5 +++
 .../cosmos.strange.love_cosmosfullnodes.yaml  |  6 ++++
 controllers/cosmosfullnode_controller.go      |  7 +++-
 internal/fullnode/build_pods.go               |  4 +--
 internal/fullnode/build_pods_test.go          |  6 ++--
 internal/fullnode/pod_control.go              |  3 +-
 internal/fullnode/pod_control_test.go         | 32 +++++++++----------
 8 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 68a5b7e3..88f5a56f 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -37,6 +37,10 @@ type FullNodeSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
+	// StartingOrdinal specifies the initial ordinal number for pod naming
+	// +kubebuilder:validation:Minimum:=0
+	StartingOrdinal *int32 `json:"startingOrdinal,omitempty"`
+
 	// Number of replicas to create.
 	// Individual replicas have a consistent identity.
 	// +kubebuilder:validation:Minimum:=0
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index a2ea16cb..4c54a4ca 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -288,6 +288,11 @@ func (in *FullNodeSnapshotStatus) DeepCopy() *FullNodeSnapshotStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FullNodeSpec) DeepCopyInto(out *FullNodeSpec) {
 	*out = *in
+	if in.StartingOrdinal != nil {
+		in, out := &in.StartingOrdinal, &out.StartingOrdinal
+		*out = new(int32)
+		**out = **in
+	}
 	in.ChainSpec.DeepCopyInto(&out.ChainSpec)
 	in.PodTemplate.DeepCopyInto(&out.PodTemplate)
 	in.RolloutStrategy.DeepCopyInto(&out.RolloutStrategy)
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index df0f8b39..67ad2d58 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -5783,6 +5783,12 @@ spec:
                         type: string
                     type: object
                 type: object
+              startingOrdinal:
+                description: StartingOrdinal specifies the initial ordinal number
+                  for pod naming
+                format: int32
+                minimum: 0
+                type: integer
               strategy:
                 description: How to scale pods when performing an update.
                 properties:
diff --git a/controllers/cosmosfullnode_controller.go b/controllers/cosmosfullnode_controller.go
index 005a955f..580aa45e 100644
--- a/controllers/cosmosfullnode_controller.go
+++ b/controllers/cosmosfullnode_controller.go
@@ -115,6 +115,11 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return stopResult, client.IgnoreNotFound(err)
 	}
 
+	startingOrdinal := int32(0)
+	if crd.Spec.StartingOrdinal != nil {
+		startingOrdinal = *crd.Spec.StartingOrdinal
+	}
+
 	reporter := kube.NewEventReporter(logger, r.recorder, crd)
 
 	fullnode.ResetStatus(crd)
@@ -174,7 +179,7 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	// Reconcile pods.
-	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo)
+	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo, startingOrdinal)
 	if err != nil {
 		errs.Append(err)
 	}
diff --git a/internal/fullnode/build_pods.go b/internal/fullnode/build_pods.go
index 8b63aec3..ff34a284 100644
--- a/internal/fullnode/build_pods.go
+++ b/internal/fullnode/build_pods.go
@@ -12,13 +12,13 @@ const (
 )
 
 // BuildPods creates the final state of pods given the crd.
-func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums) ([]diff.Resource[*corev1.Pod], error) {
+func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums, startingOrdinal int32) ([]diff.Resource[*corev1.Pod], error) {
 	var (
 		builder = NewPodBuilder(crd)
 		pods    []diff.Resource[*corev1.Pod]
 	)
 	candidates := podCandidates(crd)
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := int32(startingOrdinal); i < crd.Spec.Replicas; i++ {
 		pod, err := builder.WithOrdinal(i).Build()
 		if err != nil {
 			return nil, err
diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index bc93fcc9..1e30858b 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -38,7 +38,7 @@ func TestBuildPods(t *testing.T) {
 			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i)}] = strconv.Itoa(i)
 		}
 
-		pods, err := BuildPods(crd, cksums)
+		pods, err := BuildPods(crd, cksums, 0)
 		require.NoError(t, err)
 		require.Equal(t, 5, len(pods))
 
@@ -82,7 +82,7 @@ func TestBuildPods(t *testing.T) {
 			},
 		}
 
-		pods, err := BuildPods(crd, nil)
+		pods, err := BuildPods(crd, nil, 0)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
@@ -118,7 +118,7 @@ func TestBuildPods(t *testing.T) {
 			},
 		}
 
-		pods, err := BuildPods(crd, nil)
+		pods, err := BuildPods(crd, nil, 0)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
diff --git a/internal/fullnode/pod_control.go b/internal/fullnode/pod_control.go
index 8fc8a0d0..2a3abb2a 100644
--- a/internal/fullnode/pod_control.go
+++ b/internal/fullnode/pod_control.go
@@ -51,6 +51,7 @@ func (pc PodControl) Reconcile(
 	crd *cosmosv1.CosmosFullNode,
 	cksums ConfigChecksums,
 	syncInfo map[string]*cosmosv1.SyncInfoPodStatus,
+	startingOrdinal int32,
 ) (bool, kube.ReconcileError) {
 	var pods corev1.PodList
 	if err := pc.client.List(ctx, &pods,
@@ -60,7 +61,7 @@ func (pc PodControl) Reconcile(
 		return false, kube.TransientError(fmt.Errorf("list existing pods: %w", err))
 	}
 
-	wantPods, err := BuildPods(crd, cksums)
+	wantPods, err := BuildPods(crd, cksums, startingOrdinal)
 	if err != nil {
 		return false, kube.UnrecoverableError(fmt.Errorf("build pods: %w", err))
 	}
diff --git a/internal/fullnode/pod_control_test.go b/internal/fullnode/pod_control_test.go
index efb55df7..438d2bc4 100644
--- a/internal/fullnode/pod_control_test.go
+++ b/internal/fullnode/pod_control_test.go
@@ -69,7 +69,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		crd.Namespace = namespace
 		crd.Spec.Replicas = 1
 
-		pods, err := BuildPods(&crd, nil)
+		pods, err := BuildPods(&crd, nil, 0)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 
@@ -82,7 +82,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 
 		control := NewPodControl(mClient, nil)
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 		require.False(t, requeue)
 
@@ -108,7 +108,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		})
 
 		control := NewPodControl(mClient, nil)
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, nil)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, nil, 0)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -130,7 +130,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			MaxUnavailable: ptr(intstr.FromInt(2)),
 		}
 
-		pods, err := BuildPods(&crd, nil)
+		pods, err := BuildPods(&crd, nil, 0)
 		require.NoError(t, err)
 
 		mClient := newMockPodClient(diff.New(nil, pods).Creates())
@@ -153,7 +153,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 
 		// Trigger updates
 		crd.Spec.PodTemplate.Image = "new-image"
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -167,7 +167,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -191,7 +191,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -222,7 +222,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 		crd.Status.Height = make(map[string]uint64)
 
-		pods, err := BuildPods(&crd, nil)
+		pods, err := BuildPods(&crd, nil, 0)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 
@@ -267,7 +267,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 
 		// Reconcile 1, should update 0 and 1
 
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		// only handled 2 updates, so should requeue.
@@ -284,7 +284,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -310,7 +310,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		// no further updates yet, should requeue.
@@ -334,7 +334,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		// only handled 1 updates, so should requeue.
@@ -353,7 +353,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -381,7 +381,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		// all updates are now handled, no longer need requeue.
@@ -415,7 +415,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 		crd.Status.Height = make(map[string]uint64)
 
-		pods, err := BuildPods(&crd, nil)
+		pods, err := BuildPods(&crd, nil, 0)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 
@@ -458,7 +458,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			crd.Status.Height[pod.Name] = 100
 		}
 
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
 		require.NoError(t, err)
 
 		// all updates are handled, so should not requeue

From 212e049776556f1a1740571216f4e8ef54d4b772 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Mon, 11 Nov 2024 00:54:17 -0700
Subject: [PATCH 02/41] fix lint error

---
 internal/fullnode/build_pods.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/build_pods.go b/internal/fullnode/build_pods.go
index ff34a284..adae8bcb 100644
--- a/internal/fullnode/build_pods.go
+++ b/internal/fullnode/build_pods.go
@@ -18,7 +18,7 @@ func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums, startingOrd
 		pods    []diff.Resource[*corev1.Pod]
 	)
 	candidates := podCandidates(crd)
-	for i := int32(startingOrdinal); i < crd.Spec.Replicas; i++ {
+	for i := startingOrdinal; i < crd.Spec.Replicas; i++ {
 		pod, err := builder.WithOrdinal(i).Build()
 		if err != nil {
 			return nil, err

From 6c4f49853791537f629f47a5b109bafce1e4355c Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Mon, 11 Nov 2024 23:38:17 -0700
Subject: [PATCH 03/41] update build pods test to reflect startingordinals
 usage

---
 internal/fullnode/build_pods.go      |  2 +-
 internal/fullnode/build_pods_test.go | 40 +++++++++++++++-------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/internal/fullnode/build_pods.go b/internal/fullnode/build_pods.go
index adae8bcb..d0a69a8f 100644
--- a/internal/fullnode/build_pods.go
+++ b/internal/fullnode/build_pods.go
@@ -18,7 +18,7 @@ func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums, startingOrd
 		pods    []diff.Resource[*corev1.Pod]
 	)
 	candidates := podCandidates(crd)
-	for i := startingOrdinal; i < crd.Spec.Replicas; i++ {
+	for i := startingOrdinal; i < crd.Spec.Replicas+startingOrdinal; i++ {
 		pod, err := builder.WithOrdinal(i).Build()
 		if err != nil {
 			return nil, err
diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index 1e30858b..f69f1f32 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -17,7 +17,7 @@ import (
 func TestBuildPods(t *testing.T) {
 	t.Parallel()
 
-	t.Run("happy path", func(t *testing.T) {
+	t.Run("happy path with starting ordinal", func(t *testing.T) {
 		crd := &cosmosv1.CosmosFullNode{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "agoric",
@@ -34,36 +34,38 @@ func TestBuildPods(t *testing.T) {
 		}
 
 		cksums := make(ConfigChecksums)
+		startingOrdinal := int32(2)
 		for i := 0; i < int(crd.Spec.Replicas); i++ {
-			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i)}] = strconv.Itoa(i)
+			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(startingOrdinal))}] = strconv.Itoa(i + int(startingOrdinal))
 		}
 
-		pods, err := BuildPods(crd, cksums, 0)
+		pods, err := BuildPods(crd, cksums, startingOrdinal)
 		require.NoError(t, err)
 		require.Equal(t, 5, len(pods))
 
 		for i, r := range pods {
-			require.Equal(t, int64(i), r.Ordinal(), i)
+			expectedOrdinal := startingOrdinal + int32(i)
+			require.Equal(t, int64(expectedOrdinal), r.Ordinal(), i)
 			require.NotEmpty(t, r.Revision(), i)
-			require.Equal(t, strconv.Itoa(i), r.Object().Annotations["cosmos.strange.love/config-checksum"])
+			require.Equal(t, strconv.Itoa(int(expectedOrdinal)), r.Object().Annotations["cosmos.strange.love/config-checksum"])
 		}
 
-		want := lo.Map([]int{0, 1, 2, 3, 4}, func(_ int, i int) string {
+		want := lo.Map([]int{2, 3, 4, 5, 6}, func(i int, _ int) string {
 			return fmt.Sprintf("agoric-%d", i)
 		})
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
 		require.Equal(t, want, got)
 
-		pod, err := NewPodBuilder(crd).WithOrdinal(0).Build()
+		pod, err := NewPodBuilder(crd).WithOrdinal(startingOrdinal).Build()
 		require.NoError(t, err)
 		require.Equal(t, pod.Spec, pods[0].Object().Spec)
 	})
 
-	t.Run("instance overrides", func(t *testing.T) {
+	t.Run("instance overrides with starting ordinal", func(t *testing.T) {
 		const (
 			image         = "agoric:latest"
 			overrideImage = "some_image:custom"
-			overridePod   = "agoric-5"
+			overridePod   = "agoric-7"
 		)
 		crd := &cosmosv1.CosmosFullNode{
 			ObjectMeta: metav1.ObjectMeta{
@@ -75,18 +77,19 @@ func TestBuildPods(t *testing.T) {
 					Image: image,
 				},
 				InstanceOverrides: map[string]cosmosv1.InstanceOverridesSpec{
-					"agoric-2":  {DisableStrategy: ptr(cosmosv1.DisablePod)},
-					"agoric-4":  {DisableStrategy: ptr(cosmosv1.DisableAll)},
+					"agoric-4":  {DisableStrategy: ptr(cosmosv1.DisablePod)},
+					"agoric-6":  {DisableStrategy: ptr(cosmosv1.DisableAll)},
 					overridePod: {Image: overrideImage},
 				},
 			},
 		}
 
-		pods, err := BuildPods(crd, nil, 0)
+		startingOrdinal := int32(2)
+		pods, err := BuildPods(crd, nil, startingOrdinal)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
-		want := lo.Map([]int{0, 1, 3, 5}, func(i int, _ int) string {
+		want := lo.Map([]int{2, 3, 5, 7}, func(i int, _ int) string {
 			return fmt.Sprintf("agoric-%d", i)
 		})
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
@@ -101,7 +104,7 @@ func TestBuildPods(t *testing.T) {
 		}
 	})
 
-	t.Run("scheduled volume snapshot pod candidate", func(t *testing.T) {
+	t.Run("scheduled volume snapshot pod candidate with starting ordinal", func(t *testing.T) {
 		crd := &cosmosv1.CosmosFullNode{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: "agoric",
@@ -111,18 +114,19 @@ func TestBuildPods(t *testing.T) {
 			},
 			Status: cosmosv1.FullNodeStatus{
 				ScheduledSnapshotStatus: map[string]cosmosv1.FullNodeSnapshotStatus{
-					"some.scheduled.snapshot.1":       {PodCandidate: "agoric-1"},
-					"some.scheduled.snapshot.2":       {PodCandidate: "agoric-2"},
+					"some.scheduled.snapshot.1":       {PodCandidate: "agoric-3"},
+					"some.scheduled.snapshot.2":       {PodCandidate: "agoric-4"},
 					"some.scheduled.snapshot.ignored": {PodCandidate: "agoric-99"},
 				},
 			},
 		}
 
-		pods, err := BuildPods(crd, nil, 0)
+		startingOrdinal := int32(2)
+		pods, err := BuildPods(crd, nil, startingOrdinal)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
-		want := lo.Map([]int{0, 3, 4, 5}, func(i int, _ int) string {
+		want := lo.Map([]int{2, 5, 6, 7}, func(i int, _ int) string {
 			return fmt.Sprintf("agoric-%d", i)
 		})
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })

From 8c55a1f001cfdb6cd94ce37a60c95140ecd309cc Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Thu, 14 Nov 2024 02:38:08 -0700
Subject: [PATCH 04/41] add retries for trivy to overcome the throttling

---
 .github/workflows/release.yaml | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 91227714..772e428a 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -58,11 +58,14 @@ jobs:
           build-args: VERSION=${{ steps.meta.outputs.version }}
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: '${{ fromJSON(steps.meta.outputs.json).tags[0] }}'
-          format: 'table'
-          exit-code: '1'
-          ignore-unfixed: true
-          vuln-type: 'os,library'
-          severity: 'CRITICAL,HIGH'
+        run: |
+          for i in {1..3}; do
+            if docker run --rm aquasec/trivy:latest image --exit-code 0 --severity CRITICAL,HIGH --ignore-unfixed ${{ fromJSON(steps.meta.outputs.json).tags[0] }}; then
+              break
+            elif [ $i -lt 3 ]; then
+              echo "Retrying in 60 seconds..."
+              sleep 60
+            else
+              exit 1
+            fi
+          done
\ No newline at end of file

From 62db3553c8839ff05d1d833d7c91ed929bbf2778 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 11:37:03 -0700
Subject: [PATCH 05/41] Update: reorg so that we use a similar structure to
 stateful set instead of *imaginary* name

---
 api/v1/cosmosfullnode_types.go                | 12 ++++++---
 api/v1/zz_generated.deepcopy.go               | 26 +++++++++++++++----
 .../cosmos.strange.love_cosmosfullnodes.yaml  | 16 +++++++-----
 controllers/cosmosfullnode_controller.go      |  4 +--
 4 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 88f5a56f..0e1bae31 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -32,14 +32,20 @@ const CosmosFullNodeController = "CosmosFullNode"
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+// Ordinal specifies the configuration for pod ordinal numbers
+type Ordinal struct {
+	// Start specifies the initial ordinal number for pod naming
+	// +kubebuilder:validation:Minimum:=0
+	Start *int32 `json:"start,omitempty"`
+}
+
 // FullNodeSpec defines the desired state of CosmosFullNode
 type FullNodeSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// StartingOrdinal specifies the initial ordinal number for pod naming
-	// +kubebuilder:validation:Minimum:=0
-	StartingOrdinal *int32 `json:"startingOrdinal,omitempty"`
+	// Ordinal specifies the configuration for pod ordinal numbers
+	Ordinal Ordinal `json:"ordinal,omitempty"`
 
 	// Number of replicas to create.
 	// Individual replicas have a consistent identity.
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index 4c54a4ca..c83f6129 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -288,11 +288,7 @@ func (in *FullNodeSnapshotStatus) DeepCopy() *FullNodeSnapshotStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FullNodeSpec) DeepCopyInto(out *FullNodeSpec) {
 	*out = *in
-	if in.StartingOrdinal != nil {
-		in, out := &in.StartingOrdinal, &out.StartingOrdinal
-		*out = new(int32)
-		**out = **in
-	}
+	in.Ordinal.DeepCopyInto(&out.Ordinal)
 	in.ChainSpec.DeepCopyInto(&out.ChainSpec)
 	in.PodTemplate.DeepCopyInto(&out.PodTemplate)
 	in.RolloutStrategy.DeepCopyInto(&out.RolloutStrategy)
@@ -457,6 +453,26 @@ func (in *Metadata) DeepCopy() *Metadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Ordinal) DeepCopyInto(out *Ordinal) {
+	*out = *in
+	if in.Start != nil {
+		in, out := &in.Start, &out.Start
+		*out = new(int32)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ordinal.
+func (in *Ordinal) DeepCopy() *Ordinal {
+	if in == nil {
+		return nil
+	}
+	out := new(Ordinal)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PVCAutoScaleSpec) DeepCopyInto(out *PVCAutoScaleSpec) {
 	*out = *in
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index 67ad2d58..3dc55a10 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -555,6 +555,16 @@ spec:
                   Example: cosmos-1
                   Used for debugging.
                 type: object
+              ordinal:
+                description: Ordinal specifies the configuration for pod ordinal numbers
+                properties:
+                  start:
+                    description: Start specifies the initial ordinal number for pod
+                      naming
+                    format: int32
+                    minimum: 0
+                    type: integer
+                type: object
               podTemplate:
                 description: |-
                   Template applied to all pods.
@@ -5783,12 +5793,6 @@ spec:
                         type: string
                     type: object
                 type: object
-              startingOrdinal:
-                description: StartingOrdinal specifies the initial ordinal number
-                  for pod naming
-                format: int32
-                minimum: 0
-                type: integer
               strategy:
                 description: How to scale pods when performing an update.
                 properties:
diff --git a/controllers/cosmosfullnode_controller.go b/controllers/cosmosfullnode_controller.go
index 580aa45e..02935680 100644
--- a/controllers/cosmosfullnode_controller.go
+++ b/controllers/cosmosfullnode_controller.go
@@ -116,8 +116,8 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	startingOrdinal := int32(0)
-	if crd.Spec.StartingOrdinal != nil {
-		startingOrdinal = *crd.Spec.StartingOrdinal
+	if crd.Spec.Ordinal.Start != nil {
+		startingOrdinal = *crd.Spec.Ordinal.Start
 	}
 
 	reporter := kube.NewEventReporter(logger, r.recorder, crd)

From b9ccbfa28251306ed397295caa083878572bab4a Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:01:21 -0700
Subject: [PATCH 06/41] optimize & use starting ordinal for pvcs too

---
 api/v1/cosmosfullnode_types.go           | 2 +-
 api/v1/zz_generated.deepcopy.go          | 7 +------
 controllers/cosmosfullnode_controller.go | 7 +------
 internal/fullnode/pvc_builder.go         | 2 +-
 4 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 0e1bae31..679ec5ab 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -36,7 +36,7 @@ const CosmosFullNodeController = "CosmosFullNode"
 type Ordinal struct {
 	// Start specifies the initial ordinal number for pod naming
 	// +kubebuilder:validation:Minimum:=0
-	Start *int32 `json:"start,omitempty"`
+	Start int32 `json:"start,omitempty"`
 }
 
 // FullNodeSpec defines the desired state of CosmosFullNode
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index c83f6129..4dba169e 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -288,7 +288,7 @@ func (in *FullNodeSnapshotStatus) DeepCopy() *FullNodeSnapshotStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FullNodeSpec) DeepCopyInto(out *FullNodeSpec) {
 	*out = *in
-	in.Ordinal.DeepCopyInto(&out.Ordinal)
+	out.Ordinal = in.Ordinal
 	in.ChainSpec.DeepCopyInto(&out.ChainSpec)
 	in.PodTemplate.DeepCopyInto(&out.PodTemplate)
 	in.RolloutStrategy.DeepCopyInto(&out.RolloutStrategy)
@@ -456,11 +456,6 @@ func (in *Metadata) DeepCopy() *Metadata {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Ordinal) DeepCopyInto(out *Ordinal) {
 	*out = *in
-	if in.Start != nil {
-		in, out := &in.Start, &out.Start
-		*out = new(int32)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ordinal.
diff --git a/controllers/cosmosfullnode_controller.go b/controllers/cosmosfullnode_controller.go
index 02935680..e5a9929d 100644
--- a/controllers/cosmosfullnode_controller.go
+++ b/controllers/cosmosfullnode_controller.go
@@ -115,11 +115,6 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return stopResult, client.IgnoreNotFound(err)
 	}
 
-	startingOrdinal := int32(0)
-	if crd.Spec.Ordinal.Start != nil {
-		startingOrdinal = *crd.Spec.Ordinal.Start
-	}
-
 	reporter := kube.NewEventReporter(logger, r.recorder, crd)
 
 	fullnode.ResetStatus(crd)
@@ -179,7 +174,7 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	// Reconcile pods.
-	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo, startingOrdinal)
+	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo, crd.Spec.Ordinal.Start)
 	if err != nil {
 		errs.Append(err)
 	}
diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index b895872a..1e98b7f9 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -39,7 +39,7 @@ func BuildPVCs(
 	}
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		if pvcDisabled(crd, i) {
 			continue
 		}

From 804e076cfb22233fe7b8b1c8474fc7156e5b92f7 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:18:19 -0700
Subject: [PATCH 07/41] Update to use starting ordinal in appropriate files

---
 internal/fullnode/configmap_builder.go | 2 +-
 internal/fullnode/node_key_builder.go  | 2 +-
 internal/fullnode/peer_collector.go    | 2 +-
 internal/fullnode/pvc_control.go       | 2 +-
 internal/fullnode/service_builder.go   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index af9bbb51..74249158 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -31,7 +31,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 	defer bufPool.Put(buf)
 	defer buf.Reset()
 
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
 		instance := instanceName(crd, i)
 		if err := addConfigToml(buf, data, crd, instance, peers); err != nil {
diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index 3ed330b8..e610be90 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -21,7 +21,7 @@ const nodeKeyFile = "node_key.json"
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
 	secrets := make([]diff.Resource[*corev1.Secret], crd.Spec.Replicas)
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		var s corev1.Secret
 		s.Name = nodeKeySecretName(crd, i)
 		s.Namespace = crd.Namespace
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 37c27c3a..555ef803 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -110,7 +110,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 	if crd.Spec.Service.ClusterDomain != nil {
 		clusterDomain = *crd.Spec.Service.ClusterDomain
 	}
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
 		// Hoping the caching layer kubebuilder prevents API errors or rate limits. Simplifies logic to use a Get here
diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 6786f377..588daa2b 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -51,7 +51,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
-		for i := int32(0); i < crd.Spec.Replicas; i++ {
+		for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 			name := pvcName(crd, i)
 			found := false
 			for _, pvc := range currentPVCs {
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 2831a12a..db534553 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -32,7 +32,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
 
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		ordinal := i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)

From 75dcd8a51268ff1b8cdeb32d174b671d87e7b688 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:25:16 -0700
Subject: [PATCH 08/41] Optimize tests

---
 internal/fullnode/build_pods.go       | 4 ++--
 internal/fullnode/build_pods_test.go  | 8 +++-----
 internal/fullnode/pod_control.go      | 2 +-
 internal/fullnode/pod_control_test.go | 8 ++++----
 4 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/internal/fullnode/build_pods.go b/internal/fullnode/build_pods.go
index d0a69a8f..9b10f1b5 100644
--- a/internal/fullnode/build_pods.go
+++ b/internal/fullnode/build_pods.go
@@ -12,13 +12,13 @@ const (
 )
 
 // BuildPods creates the final state of pods given the crd.
-func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums, startingOrdinal int32) ([]diff.Resource[*corev1.Pod], error) {
+func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums) ([]diff.Resource[*corev1.Pod], error) {
 	var (
 		builder = NewPodBuilder(crd)
 		pods    []diff.Resource[*corev1.Pod]
 	)
 	candidates := podCandidates(crd)
-	for i := startingOrdinal; i < crd.Spec.Replicas+startingOrdinal; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		pod, err := builder.WithOrdinal(i).Build()
 		if err != nil {
 			return nil, err
diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index f69f1f32..6aad9886 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -39,7 +39,7 @@ func TestBuildPods(t *testing.T) {
 			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(startingOrdinal))}] = strconv.Itoa(i + int(startingOrdinal))
 		}
 
-		pods, err := BuildPods(crd, cksums, startingOrdinal)
+		pods, err := BuildPods(crd, cksums)
 		require.NoError(t, err)
 		require.Equal(t, 5, len(pods))
 
@@ -84,8 +84,7 @@ func TestBuildPods(t *testing.T) {
 			},
 		}
 
-		startingOrdinal := int32(2)
-		pods, err := BuildPods(crd, nil, startingOrdinal)
+		pods, err := BuildPods(crd, nil)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
@@ -121,8 +120,7 @@ func TestBuildPods(t *testing.T) {
 			},
 		}
 
-		startingOrdinal := int32(2)
-		pods, err := BuildPods(crd, nil, startingOrdinal)
+		pods, err := BuildPods(crd, nil)
 		require.NoError(t, err)
 		require.Equal(t, 4, len(pods))
 
diff --git a/internal/fullnode/pod_control.go b/internal/fullnode/pod_control.go
index 2a3abb2a..9fab0795 100644
--- a/internal/fullnode/pod_control.go
+++ b/internal/fullnode/pod_control.go
@@ -61,7 +61,7 @@ func (pc PodControl) Reconcile(
 		return false, kube.TransientError(fmt.Errorf("list existing pods: %w", err))
 	}
 
-	wantPods, err := BuildPods(crd, cksums, startingOrdinal)
+	wantPods, err := BuildPods(crd, cksums)
 	if err != nil {
 		return false, kube.UnrecoverableError(fmt.Errorf("build pods: %w", err))
 	}
diff --git a/internal/fullnode/pod_control_test.go b/internal/fullnode/pod_control_test.go
index 438d2bc4..f8c40f0a 100644
--- a/internal/fullnode/pod_control_test.go
+++ b/internal/fullnode/pod_control_test.go
@@ -69,7 +69,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		crd.Namespace = namespace
 		crd.Spec.Replicas = 1
 
-		pods, err := BuildPods(&crd, nil, 0)
+		pods, err := BuildPods(&crd, nil)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 
@@ -130,7 +130,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			MaxUnavailable: ptr(intstr.FromInt(2)),
 		}
 
-		pods, err := BuildPods(&crd, nil, 0)
+		pods, err := BuildPods(&crd, nil)
 		require.NoError(t, err)
 
 		mClient := newMockPodClient(diff.New(nil, pods).Creates())
@@ -222,7 +222,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 		crd.Status.Height = make(map[string]uint64)
 
-		pods, err := BuildPods(&crd, nil, 0)
+		pods, err := BuildPods(&crd, nil)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 
@@ -415,7 +415,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 		crd.Status.Height = make(map[string]uint64)
 
-		pods, err := BuildPods(&crd, nil, 0)
+		pods, err := BuildPods(&crd, nil)
 		require.NoError(t, err)
 		existing := diff.New(nil, pods).Creates()
 

From bfe1a51dd9704aa8a3c27efba57738ce35279b81 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:31:03 -0700
Subject: [PATCH 09/41] fix test

---
 internal/fullnode/build_pods_test.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index 6aad9886..ae73f85a 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -30,13 +30,15 @@ func TestBuildPods(t *testing.T) {
 					Image: "busybox:latest",
 				},
 				InstanceOverrides: nil,
+				Ordinal: cosmosv1.Ordinal{
+					Start: 2,
+				},
 			},
 		}
 
 		cksums := make(ConfigChecksums)
-		startingOrdinal := int32(2)
 		for i := 0; i < int(crd.Spec.Replicas); i++ {
-			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(startingOrdinal))}] = strconv.Itoa(i + int(startingOrdinal))
+			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(crd.Spec.Ordinal.Start))}] = strconv.Itoa(i + int(crd.Spec.Ordinal.Start))
 		}
 
 		pods, err := BuildPods(crd, cksums)
@@ -44,7 +46,7 @@ func TestBuildPods(t *testing.T) {
 		require.Equal(t, 5, len(pods))
 
 		for i, r := range pods {
-			expectedOrdinal := startingOrdinal + int32(i)
+			expectedOrdinal := crd.Spec.Ordinal.Start + int32(i)
 			require.Equal(t, int64(expectedOrdinal), r.Ordinal(), i)
 			require.NotEmpty(t, r.Revision(), i)
 			require.Equal(t, strconv.Itoa(int(expectedOrdinal)), r.Object().Annotations["cosmos.strange.love/config-checksum"])
@@ -56,7 +58,7 @@ func TestBuildPods(t *testing.T) {
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
 		require.Equal(t, want, got)
 
-		pod, err := NewPodBuilder(crd).WithOrdinal(startingOrdinal).Build()
+		pod, err := NewPodBuilder(crd).WithOrdinal(crd.Spec.Ordinal.Start).Build()
 		require.NoError(t, err)
 		require.Equal(t, pod.Spec, pods[0].Object().Spec)
 	})

From e088656d6508d6e60434a267907a7ac7e262d4b0 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:38:43 -0700
Subject: [PATCH 10/41] fix test

---
 internal/fullnode/build_pods_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index ae73f85a..a569567e 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -83,6 +83,9 @@ func TestBuildPods(t *testing.T) {
 					"agoric-6":  {DisableStrategy: ptr(cosmosv1.DisableAll)},
 					overridePod: {Image: overrideImage},
 				},
+				Ordinal: cosmosv1.Ordinal{
+					Start: 2,
+				},
 			},
 		}
 
@@ -120,6 +123,9 @@ func TestBuildPods(t *testing.T) {
 					"some.scheduled.snapshot.ignored": {PodCandidate: "agoric-99"},
 				},
 			},
+			Ordinal: cosmosv1.Ordinal{
+				Start: 2,
+			},
 		}
 
 		pods, err := BuildPods(crd, nil)

From 0a52ff48fd53c5fc79639062692742832fd5ac2a Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:41:39 -0700
Subject: [PATCH 11/41] Use Ordinal within spec (duh))

---
 internal/fullnode/build_pods_test.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index a569567e..66e1ac56 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -115,6 +115,7 @@ func TestBuildPods(t *testing.T) {
 			},
 			Spec: cosmosv1.FullNodeSpec{
 				Replicas: 6,
+				Ordinal:  cosmosv1.Ordinal{Start: 2},
 			},
 			Status: cosmosv1.FullNodeStatus{
 				ScheduledSnapshotStatus: map[string]cosmosv1.FullNodeSnapshotStatus{
@@ -123,9 +124,6 @@ func TestBuildPods(t *testing.T) {
 					"some.scheduled.snapshot.ignored": {PodCandidate: "agoric-99"},
 				},
 			},
-			Ordinal: cosmosv1.Ordinal{
-				Start: 2,
-			},
 		}
 
 		pods, err := BuildPods(crd, nil)

From 91bded5881d2ff2d52261035d045f9038206af26 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 22:47:25 -0700
Subject: [PATCH 12/41] Remove redundant parameters

---
 controllers/cosmosfullnode_controller.go |  2 +-
 internal/fullnode/pod_control.go         |  1 -
 internal/fullnode/pod_control_test.go    | 24 ++++++++++++------------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/controllers/cosmosfullnode_controller.go b/controllers/cosmosfullnode_controller.go
index e5a9929d..005a955f 100644
--- a/controllers/cosmosfullnode_controller.go
+++ b/controllers/cosmosfullnode_controller.go
@@ -174,7 +174,7 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	// Reconcile pods.
-	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo, crd.Spec.Ordinal.Start)
+	podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums, syncInfo)
 	if err != nil {
 		errs.Append(err)
 	}
diff --git a/internal/fullnode/pod_control.go b/internal/fullnode/pod_control.go
index 9fab0795..8fc8a0d0 100644
--- a/internal/fullnode/pod_control.go
+++ b/internal/fullnode/pod_control.go
@@ -51,7 +51,6 @@ func (pc PodControl) Reconcile(
 	crd *cosmosv1.CosmosFullNode,
 	cksums ConfigChecksums,
 	syncInfo map[string]*cosmosv1.SyncInfoPodStatus,
-	startingOrdinal int32,
 ) (bool, kube.ReconcileError) {
 	var pods corev1.PodList
 	if err := pc.client.List(ctx, &pods,
diff --git a/internal/fullnode/pod_control_test.go b/internal/fullnode/pod_control_test.go
index f8c40f0a..efb55df7 100644
--- a/internal/fullnode/pod_control_test.go
+++ b/internal/fullnode/pod_control_test.go
@@ -82,7 +82,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		}
 
 		control := NewPodControl(mClient, nil)
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 		require.False(t, requeue)
 
@@ -108,7 +108,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 		})
 
 		control := NewPodControl(mClient, nil)
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, nil, 0)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, nil)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -153,7 +153,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 
 		// Trigger updates
 		crd.Spec.PodTemplate.Image = "new-image"
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -167,7 +167,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -191,7 +191,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 		require.True(t, requeue)
 
@@ -267,7 +267,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 
 		// Reconcile 1, should update 0 and 1
 
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		// only handled 2 updates, so should requeue.
@@ -284,7 +284,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -310,7 +310,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		// no further updates yet, should requeue.
@@ -334,7 +334,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		// only handled 1 updates, so should requeue.
@@ -353,7 +353,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		require.True(t, requeue)
@@ -381,7 +381,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			return kube.ComputeRollout(maxUnavail, desired, ready)
 		}
 
-		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err = control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		// all updates are now handled, no longer need requeue.
@@ -458,7 +458,7 @@ func TestPodControl_Reconcile(t *testing.T) {
 			crd.Status.Height[pod.Name] = 100
 		}
 
-		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo, 0)
+		requeue, err := control.Reconcile(ctx, nopReporter, &crd, nil, syncInfo)
 		require.NoError(t, err)
 
 		// all updates are handled, so should not requeue

From 6bb42dd5170b7535737f3177489e9da53737632a Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 23:09:42 -0700
Subject: [PATCH 13/41] Note: this is incorrect since PVC allocations start
 from 0, only the naming will start from spec.ordinal.start

---
 internal/fullnode/pvc_control.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 588daa2b..6786f377 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -51,7 +51,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
-		for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+		for i := int32(0); i < crd.Spec.Replicas; i++ {
 			name := pvcName(crd, i)
 			found := false
 			for _, pvc := range currentPVCs {

From a8cc9e0b3711aa64df12bc425e25f06fc95de730 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 19 Nov 2024 23:33:58 -0700
Subject: [PATCH 14/41] Attempt to find root cause of mapping issue crash

---
 internal/fullnode/configmap_builder.go | 2 +-
 internal/fullnode/node_key_builder.go  | 2 +-
 internal/fullnode/peer_collector.go    | 2 +-
 internal/fullnode/service_builder.go   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index 74249158..af9bbb51 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -31,7 +31,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 	defer bufPool.Put(buf)
 	defer buf.Reset()
 
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
 		instance := instanceName(crd, i)
 		if err := addConfigToml(buf, data, crd, instance, peers); err != nil {
diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index e610be90..3ed330b8 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -21,7 +21,7 @@ const nodeKeyFile = "node_key.json"
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
 	secrets := make([]diff.Resource[*corev1.Secret], crd.Spec.Replicas)
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		var s corev1.Secret
 		s.Name = nodeKeySecretName(crd, i)
 		s.Namespace = crd.Namespace
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 555ef803..37c27c3a 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -110,7 +110,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 	if crd.Spec.Service.ClusterDomain != nil {
 		clusterDomain = *crd.Spec.Service.ClusterDomain
 	}
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
 		// Hoping the caching layer kubebuilder prevents API errors or rate limits. Simplifies logic to use a Get here
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index db534553..2831a12a 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -32,7 +32,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
 
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		ordinal := i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)

From be87c7d852b55e962d1d97d1aa1b99f2bfb97b54 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:06:49 -0700
Subject: [PATCH 15/41] Adding spec.ordinal.start usage back to
 configmap_builder.go

---
 internal/fullnode/configmap_builder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index af9bbb51..74249158 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -31,7 +31,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 	defer bufPool.Put(buf)
 	defer buf.Reset()
 
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
 		instance := instanceName(crd, i)
 		if err := addConfigToml(buf, data, crd, instance, peers); err != nil {

From e62a9a964792e5d17ce7c2fe3b8883871cbaba5d Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:15:51 -0700
Subject: [PATCH 16/41] Adding spec.ordinal.start usage back to
 node_keybuilder.go

---
 internal/fullnode/configmap_builder.go | 2 +-
 internal/fullnode/node_key_builder.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index 74249158..af9bbb51 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -31,7 +31,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 	defer bufPool.Put(buf)
 	defer buf.Reset()
 
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
 		instance := instanceName(crd, i)
 		if err := addConfigToml(buf, data, crd, instance, peers); err != nil {
diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index 3ed330b8..e610be90 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -21,7 +21,7 @@ const nodeKeyFile = "node_key.json"
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
 	secrets := make([]diff.Resource[*corev1.Secret], crd.Spec.Replicas)
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		var s corev1.Secret
 		s.Name = nodeKeySecretName(crd, i)
 		s.Namespace = crd.Namespace

From 5905c9ba56892757a8ff6e4645bda4602d9201cc Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:28:34 -0700
Subject: [PATCH 17/41] Adding spec.ordinal.start usage back to
 peer_collector.go

---
 internal/fullnode/node_key_builder.go | 2 +-
 internal/fullnode/peer_collector.go   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index e610be90..3ed330b8 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -21,7 +21,7 @@ const nodeKeyFile = "node_key.json"
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
 	secrets := make([]diff.Resource[*corev1.Secret], crd.Spec.Replicas)
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		var s corev1.Secret
 		s.Name = nodeKeySecretName(crd, i)
 		s.Namespace = crd.Namespace
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 37c27c3a..555ef803 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -110,7 +110,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 	if crd.Spec.Service.ClusterDomain != nil {
 		clusterDomain = *crd.Spec.Service.ClusterDomain
 	}
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
 		// Hoping the caching layer kubebuilder prevents API errors or rate limits. Simplifies logic to use a Get here

From 3b27cf11c678431ff4e56ec9a75fda72510ed0a5 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:36:27 -0700
Subject: [PATCH 18/41] Add spec.ordinal.start usage back to service_builder.go

---
 internal/fullnode/peer_collector.go  | 2 +-
 internal/fullnode/service_builder.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 555ef803..37c27c3a 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -110,7 +110,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 	if crd.Spec.Service.ClusterDomain != nil {
 		clusterDomain = *crd.Spec.Service.ClusterDomain
 	}
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
 		// Hoping the caching layer kubebuilder prevents API errors or rate limits. Simplifies logic to use a Get here
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 2831a12a..db534553 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -32,7 +32,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
 
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 		ordinal := i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)

From 65c1d33f364e8fbb268c6f99225220ea15743b0c Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 00:44:17 -0700
Subject: [PATCH 19/41] service_builder.go

---
 internal/fullnode/service_builder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index db534553..2831a12a 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -32,7 +32,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
 
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		ordinal := i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)

From 51747d4cb2c901676d4fc728135007df55901b49 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 01:02:10 -0700
Subject: [PATCH 20/41] Update pvc to use name from ordinals , but resource
 from 0

---
 internal/fullnode/pvc_builder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index 1e98b7f9..3519152e 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -51,7 +51,7 @@ func BuildPVCs(
 
 		var dataSource *corev1.TypedLocalObjectReference
 		var existingSize resource.Quantity
-		if ds, ok := dataSources[i]; ok && ds != nil {
+		if ds, ok := dataSources[i-crd.Spec.Ordinal.Start]; ok && ds != nil {
 			dataSource = ds.ref
 		} else {
 			for _, pvc := range currentPVCs {

From df5a24233f521484b205847d7b842da522ca4dd1 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 01:14:33 -0700
Subject: [PATCH 21/41] revert previous commit

---
 internal/fullnode/pvc_builder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index 3519152e..1e98b7f9 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -51,7 +51,7 @@ func BuildPVCs(
 
 		var dataSource *corev1.TypedLocalObjectReference
 		var existingSize resource.Quantity
-		if ds, ok := dataSources[i-crd.Spec.Ordinal.Start]; ok && ds != nil {
+		if ds, ok := dataSources[i]; ok && ds != nil {
 			dataSource = ds.ref
 		} else {
 			for _, pvc := range currentPVCs {

From 8be6661a6cea7b033806fdd2b2da4fddba08e2dc Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 01:20:51 -0700
Subject: [PATCH 22/41] Update pvc_contro.go to use ordinal start

---
 internal/fullnode/pvc_control.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 6786f377..588daa2b 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -51,7 +51,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
-		for i := int32(0); i < crd.Spec.Replicas; i++ {
+		for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
 			name := pvcName(crd, i)
 			found := false
 			for _, pvc := range currentPVCs {

From dc23a15c3ad034f7358bf4051d40f2a15ae4eae3 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 01:55:47 -0700
Subject: [PATCH 23/41] name/label update

---
 internal/fullnode/pvc_builder.go | 6 +++---
 internal/fullnode/pvc_control.go | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index 1e98b7f9..abf07b1a 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -39,15 +39,15 @@ func BuildPVCs(
 	}
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		if pvcDisabled(crd, i) {
 			continue
 		}
 
 		pvc := base.DeepCopy()
-		name := pvcName(crd, i)
+		name := pvcName(crd, i+crd.Spec.Ordinal.Start)
 		pvc.Name = name
-		pvc.Labels[kube.InstanceLabel] = instanceName(crd, i)
+		pvc.Labels[kube.InstanceLabel] = instanceName(crd, i+crd.Spec.Ordinal.Start)
 
 		var dataSource *corev1.TypedLocalObjectReference
 		var existingSize resource.Quantity
diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 588daa2b..6786f377 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -51,7 +51,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
-		for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+		for i := int32(0); i < crd.Spec.Replicas; i++ {
 			name := pvcName(crd, i)
 			found := false
 			for _, pvc := range currentPVCs {

From 060d6cf3d22324e67648199d29ae006fbe3284ee Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 03:22:53 -0700
Subject: [PATCH 24/41] Attempting range check

---
 internal/fullnode/pod_builder.go | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/internal/fullnode/pod_builder.go b/internal/fullnode/pod_builder.go
index 92ba5c2a..6d5a8d22 100644
--- a/internal/fullnode/pod_builder.go
+++ b/internal/fullnode/pod_builder.go
@@ -232,13 +232,17 @@ const (
 // ordered sequence. Pods have deterministic, consistent names similar to a StatefulSet instead of generated names.
 func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 	pod := b.pod.DeepCopy()
-	name := instanceName(b.crd, ordinal)
 
-	pod.Labels[kube.InstanceLabel] = name
+	// Validate ordinal is within valid range based on start ordinal
+	startOrdinal := b.crd.Spec.Ordinal.Start
+	if ordinal < startOrdinal || ordinal >= (startOrdinal+b.crd.Spec.Replicas) {
+		return b
+	}
 
+	name := instanceName(b.crd, ordinal)
+	pod.Labels[kube.InstanceLabel] = name
 	pod.Name = name
 	pod.Spec.InitContainers = initContainers(b.crd, name)
-
 	pod.Spec.Hostname = pod.Name
 	pod.Spec.Subdomain = b.crd.Name
 
@@ -246,7 +250,9 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		{
 			Name: volChainHome,
 			VolumeSource: corev1.VolumeSource{
-				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName(b.crd, ordinal)},
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+					ClaimName: pvcName(b.crd, ordinal),
+				},
 			},
 		},
 		{
@@ -259,7 +265,9 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 			Name: volConfig,
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{Name: instanceName(b.crd, ordinal)},
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: instanceName(b.crd, ordinal),
+					},
 					Items: []corev1.KeyToPath{
 						{Key: configOverlayFile, Path: configOverlayFile},
 						{Key: appOverlayFile, Path: appOverlayFile},
@@ -286,12 +294,11 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		},
 	}
 
-	// Mounts required by all containers.
 	mounts := []corev1.VolumeMount{
 		{Name: volChainHome, MountPath: ChainHomeDir(b.crd)},
 		{Name: volSystemTmp, MountPath: systemTmpDir},
 	}
-	// Additional mounts only needed for init containers.
+
 	for i := range pod.Spec.InitContainers {
 		pod.Spec.InitContainers[i].VolumeMounts = append(mounts, []corev1.VolumeMount{
 			{Name: volTmp, MountPath: tmpDir},
@@ -299,14 +306,16 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		}...)
 	}
 
-	// At this point, guaranteed to have at least 2 containers.
 	pod.Spec.Containers[0].VolumeMounts = append(mounts, corev1.VolumeMount{
-		Name: volNodeKey, MountPath: path.Join(ChainHomeDir(b.crd), "config", nodeKeyFile), SubPath: nodeKeyFile,
+		Name:      volNodeKey,
+		MountPath: path.Join(ChainHomeDir(b.crd), "config", nodeKeyFile),
+		SubPath:   nodeKeyFile,
 	})
+
 	pod.Spec.Containers[1].VolumeMounts = []corev1.VolumeMount{
-		// The healthcheck sidecar needs access to the home directory so it can read disk usage.
 		{Name: volChainHome, MountPath: ChainHomeDir(b.crd), ReadOnly: true},
 	}
+
 	if len(pod.Spec.Containers) > 2 {
 		pod.Spec.Containers[2].VolumeMounts = mounts
 	}

From 4a116672769acc50d26855724d76ef138b6d7b42 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 03:26:46 -0700
Subject: [PATCH 25/41] Revert previous

---
 internal/fullnode/pod_builder.go | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/internal/fullnode/pod_builder.go b/internal/fullnode/pod_builder.go
index 6d5a8d22..92ba5c2a 100644
--- a/internal/fullnode/pod_builder.go
+++ b/internal/fullnode/pod_builder.go
@@ -232,17 +232,13 @@ const (
 // ordered sequence. Pods have deterministic, consistent names similar to a StatefulSet instead of generated names.
 func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 	pod := b.pod.DeepCopy()
-
-	// Validate ordinal is within valid range based on start ordinal
-	startOrdinal := b.crd.Spec.Ordinal.Start
-	if ordinal < startOrdinal || ordinal >= (startOrdinal+b.crd.Spec.Replicas) {
-		return b
-	}
-
 	name := instanceName(b.crd, ordinal)
+
 	pod.Labels[kube.InstanceLabel] = name
+
 	pod.Name = name
 	pod.Spec.InitContainers = initContainers(b.crd, name)
+
 	pod.Spec.Hostname = pod.Name
 	pod.Spec.Subdomain = b.crd.Name
 
@@ -250,9 +246,7 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		{
 			Name: volChainHome,
 			VolumeSource: corev1.VolumeSource{
-				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-					ClaimName: pvcName(b.crd, ordinal),
-				},
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName(b.crd, ordinal)},
 			},
 		},
 		{
@@ -265,9 +259,7 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 			Name: volConfig,
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: instanceName(b.crd, ordinal),
-					},
+					LocalObjectReference: corev1.LocalObjectReference{Name: instanceName(b.crd, ordinal)},
 					Items: []corev1.KeyToPath{
 						{Key: configOverlayFile, Path: configOverlayFile},
 						{Key: appOverlayFile, Path: appOverlayFile},
@@ -294,11 +286,12 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		},
 	}
 
+	// Mounts required by all containers.
 	mounts := []corev1.VolumeMount{
 		{Name: volChainHome, MountPath: ChainHomeDir(b.crd)},
 		{Name: volSystemTmp, MountPath: systemTmpDir},
 	}
-
+	// Additional mounts only needed for init containers.
 	for i := range pod.Spec.InitContainers {
 		pod.Spec.InitContainers[i].VolumeMounts = append(mounts, []corev1.VolumeMount{
 			{Name: volTmp, MountPath: tmpDir},
@@ -306,16 +299,14 @@ func (b PodBuilder) WithOrdinal(ordinal int32) PodBuilder {
 		}...)
 	}
 
+	// At this point, guaranteed to have at least 2 containers.
 	pod.Spec.Containers[0].VolumeMounts = append(mounts, corev1.VolumeMount{
-		Name:      volNodeKey,
-		MountPath: path.Join(ChainHomeDir(b.crd), "config", nodeKeyFile),
-		SubPath:   nodeKeyFile,
+		Name: volNodeKey, MountPath: path.Join(ChainHomeDir(b.crd), "config", nodeKeyFile), SubPath: nodeKeyFile,
 	})
-
 	pod.Spec.Containers[1].VolumeMounts = []corev1.VolumeMount{
+		// The healthcheck sidecar needs access to the home directory so it can read disk usage.
 		{Name: volChainHome, MountPath: ChainHomeDir(b.crd), ReadOnly: true},
 	}
-
 	if len(pod.Spec.Containers) > 2 {
 		pod.Spec.Containers[2].VolumeMounts = mounts
 	}

From 3c8739ef18bb12111ed787cf648bd369ca0c4133 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 04:06:51 -0700
Subject: [PATCH 26/41] Update configmap , key peer and service builders to use
 correct ordinals values

---
 internal/fullnode/configmap_builder.go | 7 ++++---
 internal/fullnode/node_key_builder.go  | 8 +++++---
 internal/fullnode/peer_collector.go    | 4 +++-
 internal/fullnode/service_builder.go   | 5 +++--
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index af9bbb51..45aa0f0e 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -26,12 +26,13 @@ const (
 func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource[*corev1.ConfigMap], error) {
 	var (
 		buf = bufPool.Get().(*bytes.Buffer)
-		cms = make([]diff.Resource[*corev1.ConfigMap], crd.Spec.Replicas)
+		cms = make([]diff.Resource[*corev1.ConfigMap], 0, crd.Spec.Replicas)
 	)
 	defer bufPool.Put(buf)
 	defer buf.Reset()
+	startOrdinal := crd.Spec.Ordinal.Start
 
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	for i := startOrdinal; i < startOrdinal+crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
 		instance := instanceName(crd, i)
 		if err := addConfigToml(buf, data, crd, instance, peers); err != nil {
@@ -75,7 +76,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 		)
 		cm.Data = data
 		kube.NormalizeMetadata(&cm.ObjectMeta)
-		cms[i] = diff.Adapt(&cm, i)
+		cms = append(cms, diff.Adapt(&cm, int(i-startOrdinal)))
 	}
 
 	return cms, nil
diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index 3ed330b8..bc2a8caf 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -20,8 +20,10 @@ const nodeKeyFile = "node_key.json"
 // If the secret already has a node key, it is reused.
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
-	secrets := make([]diff.Resource[*corev1.Secret], crd.Spec.Replicas)
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+	secrets := make([]diff.Resource[*corev1.Secret], 0, crd.Spec.Replicas)
+	startOrdinal := crd.Spec.Ordinal.Start
+
+	for i := startOrdinal; i < startOrdinal+crd.Spec.Replicas; i++ {
 		var s corev1.Secret
 		s.Name = nodeKeySecretName(crd, i)
 		s.Namespace = crd.Namespace
@@ -52,7 +54,7 @@ func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode
 			}
 		}
 
-		secrets[i] = diff.Adapt(&secret, i)
+		secrets = append(secrets, diff.Adapt(&secret, int(i-startOrdinal)))
 	}
 	return secrets, nil
 }
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 37c27c3a..4247dba2 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -105,12 +105,14 @@ func NewPeerCollector(client Getter) *PeerCollector {
 // Collect peer information given the crd.
 func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode) (Peers, kube.ReconcileError) {
 	peers := make(Peers)
+	startOrdinal := crd.Spec.Ordinal.Start
 
 	clusterDomain := "cluster.local"
 	if crd.Spec.Service.ClusterDomain != nil {
 		clusterDomain = *crd.Spec.Service.ClusterDomain
 	}
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
+
+	for i := startOrdinal; i < startOrdinal+crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
 		// Hoping the caching layer kubebuilder prevents API errors or rate limits. Simplifies logic to use a Get here
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 2831a12a..0c9aad32 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -31,9 +31,10 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	}
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
+	startOrdinal := crd.Spec.Ordinal.Start
 
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		ordinal := i
+		ordinal := startOrdinal + i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)
 		svc.Namespace = crd.Namespace
@@ -66,7 +67,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			svc.Spec.ClusterIP = *valOrDefault(crd.Spec.Service.P2PTemplate.ClusterIP, ptr(""))
 		}
 
-		p2ps[i] = diff.Adapt(&svc, i)
+		p2ps[i] = diff.Adapt(&svc, int(i))
 	}
 
 	rpc := rpcService(crd)

From b124b07fd1717a5db475da079d004c2996325c56 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 27 Nov 2024 00:46:49 -0700
Subject: [PATCH 27/41] Allow instanceoverrides for pvcs (example: changing 
 storage)  to work with ordinals

---
 internal/fullnode/pvc_builder.go | 36 +++++++++++++++++---------------
 internal/fullnode/pvc_control.go |  8 ++++---
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index abf07b1a..226f2210 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -40,14 +40,16 @@ func BuildPVCs(
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		if pvcDisabled(crd, i) {
+		ordinal := i + crd.Spec.Ordinal.Start
+		if pvcDisabled(crd, ordinal) {
 			continue
 		}
 
 		pvc := base.DeepCopy()
-		name := pvcName(crd, i+crd.Spec.Ordinal.Start)
+		name := pvcName(crd, ordinal)
 		pvc.Name = name
-		pvc.Labels[kube.InstanceLabel] = instanceName(crd, i+crd.Spec.Ordinal.Start)
+		podName := instanceName(crd, ordinal)
+		pvc.Labels[kube.InstanceLabel] = podName
 
 		var dataSource *corev1.TypedLocalObjectReference
 		var existingSize resource.Quantity
@@ -65,7 +67,7 @@ func BuildPVCs(
 		}
 
 		tpl := crd.Spec.VolumeClaimTemplate
-		if override, ok := crd.Spec.InstanceOverrides[instanceName(crd, i)]; ok {
+		if override, ok := crd.Spec.InstanceOverrides[podName]; ok {
 			if overrideTpl := override.VolumeClaimTemplate; overrideTpl != nil {
 				tpl = *overrideTpl
 			}
@@ -73,7 +75,7 @@ func BuildPVCs(
 
 		pvc.Spec = corev1.PersistentVolumeClaimSpec{
 			AccessModes:      sliceOrDefault(tpl.AccessModes, defaultAccessModes),
-			Resources:        pvcResources(crd, name, dataSources[i], existingSize),
+			Resources:        pvcResources(crd, name, dataSources[i], existingSize, tpl.Resources),
 			StorageClassName: ptr(tpl.StorageClassName),
 			VolumeMode:       valOrDefault(tpl.VolumeMode, ptr(corev1.PersistentVolumeFilesystem)),
 		}
@@ -88,24 +90,14 @@ func BuildPVCs(
 	return pvcs
 }
 
-func pvcDisabled(crd *cosmosv1.CosmosFullNode, ordinal int32) bool {
-	name := instanceName(crd, ordinal)
-	disable := crd.Spec.InstanceOverrides[name].DisableStrategy
-	return disable != nil && *disable == cosmosv1.DisableAll
-}
-
-func pvcName(crd *cosmosv1.CosmosFullNode, ordinal int32) string {
-	name := fmt.Sprintf("pvc-%s-%d", appName(crd), ordinal)
-	return kube.ToName(name)
-}
-
 func pvcResources(
 	crd *cosmosv1.CosmosFullNode,
 	name string,
 	dataSource *dataSource,
 	existingSize resource.Quantity,
+	tplResources corev1.ResourceRequirements,
 ) corev1.ResourceRequirements {
-	var reqs = crd.Spec.VolumeClaimTemplate.Resources.DeepCopy()
+	reqs := tplResources.DeepCopy()
 
 	if dataSource != nil {
 		reqs.Requests[corev1.ResourceStorage] = dataSource.size
@@ -129,3 +121,13 @@ func pvcResources(
 
 	return *reqs
 }
+func pvcDisabled(crd *cosmosv1.CosmosFullNode, ordinal int32) bool {
+	name := instanceName(crd, ordinal)
+	disable := crd.Spec.InstanceOverrides[name].DisableStrategy
+	return disable != nil && *disable == cosmosv1.DisableAll
+}
+
+func pvcName(crd *cosmosv1.CosmosFullNode, ordinal int32) string {
+	name := fmt.Sprintf("pvc-%s-%d", appName(crd), ordinal)
+	return kube.ToName(name)
+}
diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 6786f377..7ba8c058 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -52,7 +52,8 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
 		for i := int32(0); i < crd.Spec.Replicas; i++ {
-			name := pvcName(crd, i)
+			ordinal := i + crd.Spec.Ordinal.Start
+			name := pvcName(crd, ordinal)
 			found := false
 			for _, pvc := range currentPVCs {
 				if pvc.Name == name {
@@ -61,7 +62,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 				}
 			}
 			if !found {
-				ds := control.findDataSource(ctx, reporter, crd, i)
+				ds := control.findDataSource(ctx, reporter, crd, ordinal)
 				if ds == nil {
 					ds = &dataSource{
 						size: crd.Spec.VolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage],
@@ -160,7 +161,8 @@ type dataSource struct {
 }
 
 func (control PVCControl) findDataSource(ctx context.Context, reporter kube.Reporter, crd *cosmosv1.CosmosFullNode, ordinal int32) *dataSource {
-	if override, ok := crd.Spec.InstanceOverrides[instanceName(crd, ordinal)]; ok {
+	podName := instanceName(crd, ordinal)
+	if override, ok := crd.Spec.InstanceOverrides[podName]; ok {
 		if overrideTpl := override.VolumeClaimTemplate; overrideTpl != nil {
 			return control.findDataSourceWithPvcSpec(ctx, reporter, crd, *overrideTpl, ordinal)
 		}

From ae29f8a06858d2bd3df4fd6216e51fb20d9e83ff Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Mon, 2 Dec 2024 22:32:53 -0700
Subject: [PATCH 28/41] Use 'Ordinals' instead of 'Ordinals' as per review

---
 api/v1/cosmosfullnode_types.go         |  4 ++--
 api/v1/zz_generated.deepcopy.go        | 10 +++++-----
 internal/fullnode/build_pods.go        |  2 +-
 internal/fullnode/build_pods_test.go   | 12 ++++++------
 internal/fullnode/configmap_builder.go |  2 +-
 internal/fullnode/node_key_builder.go  |  2 +-
 internal/fullnode/peer_collector.go    |  2 +-
 internal/fullnode/pvc_builder.go       |  2 +-
 internal/fullnode/pvc_control.go       |  2 +-
 internal/fullnode/service_builder.go   |  2 +-
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 679ec5ab..9b869af6 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -33,7 +33,7 @@ const CosmosFullNodeController = "CosmosFullNode"
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 // Ordinal specifies the configuration for pod ordinal numbers
-type Ordinal struct {
+type Ordinals struct {
 	// Start specifies the initial ordinal number for pod naming
 	// +kubebuilder:validation:Minimum:=0
 	Start int32 `json:"start,omitempty"`
@@ -45,7 +45,7 @@ type FullNodeSpec struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	// Ordinal specifies the configuration for pod ordinal numbers
-	Ordinal Ordinal `json:"ordinal,omitempty"`
+	Ordinals Ordinals `json:"ordinal,omitempty"`
 
 	// Number of replicas to create.
 	// Individual replicas have a consistent identity.
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index 4dba169e..1b11a80f 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -288,7 +288,7 @@ func (in *FullNodeSnapshotStatus) DeepCopy() *FullNodeSnapshotStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FullNodeSpec) DeepCopyInto(out *FullNodeSpec) {
 	*out = *in
-	out.Ordinal = in.Ordinal
+	out.Ordinals = in.Ordinals
 	in.ChainSpec.DeepCopyInto(&out.ChainSpec)
 	in.PodTemplate.DeepCopyInto(&out.PodTemplate)
 	in.RolloutStrategy.DeepCopyInto(&out.RolloutStrategy)
@@ -454,16 +454,16 @@ func (in *Metadata) DeepCopy() *Metadata {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Ordinal) DeepCopyInto(out *Ordinal) {
+func (in *Ordinals) DeepCopyInto(out *Ordinals) {
 	*out = *in
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ordinal.
-func (in *Ordinal) DeepCopy() *Ordinal {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ordinals.
+func (in *Ordinals) DeepCopy() *Ordinals {
 	if in == nil {
 		return nil
 	}
-	out := new(Ordinal)
+	out := new(Ordinals)
 	in.DeepCopyInto(out)
 	return out
 }
diff --git a/internal/fullnode/build_pods.go b/internal/fullnode/build_pods.go
index 9b10f1b5..370ff3cd 100644
--- a/internal/fullnode/build_pods.go
+++ b/internal/fullnode/build_pods.go
@@ -18,7 +18,7 @@ func BuildPods(crd *cosmosv1.CosmosFullNode, cksums ConfigChecksums) ([]diff.Res
 		pods    []diff.Resource[*corev1.Pod]
 	)
 	candidates := podCandidates(crd)
-	for i := crd.Spec.Ordinal.Start; i < crd.Spec.Ordinal.Start+crd.Spec.Replicas; i++ {
+	for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
 		pod, err := builder.WithOrdinal(i).Build()
 		if err != nil {
 			return nil, err
diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index 66e1ac56..732b45af 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -30,7 +30,7 @@ func TestBuildPods(t *testing.T) {
 					Image: "busybox:latest",
 				},
 				InstanceOverrides: nil,
-				Ordinal: cosmosv1.Ordinal{
+				Ordinals: cosmosv1.Ordinals{
 					Start: 2,
 				},
 			},
@@ -38,7 +38,7 @@ func TestBuildPods(t *testing.T) {
 
 		cksums := make(ConfigChecksums)
 		for i := 0; i < int(crd.Spec.Replicas); i++ {
-			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(crd.Spec.Ordinal.Start))}] = strconv.Itoa(i + int(crd.Spec.Ordinal.Start))
+			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(crd.Spec.Ordinals.Start))}] = strconv.Itoa(i + int(crd.Spec.Ordinals.Start))
 		}
 
 		pods, err := BuildPods(crd, cksums)
@@ -46,7 +46,7 @@ func TestBuildPods(t *testing.T) {
 		require.Equal(t, 5, len(pods))
 
 		for i, r := range pods {
-			expectedOrdinal := crd.Spec.Ordinal.Start + int32(i)
+			expectedOrdinal := crd.Spec.Ordinals.Start + int32(i)
 			require.Equal(t, int64(expectedOrdinal), r.Ordinal(), i)
 			require.NotEmpty(t, r.Revision(), i)
 			require.Equal(t, strconv.Itoa(int(expectedOrdinal)), r.Object().Annotations["cosmos.strange.love/config-checksum"])
@@ -58,7 +58,7 @@ func TestBuildPods(t *testing.T) {
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
 		require.Equal(t, want, got)
 
-		pod, err := NewPodBuilder(crd).WithOrdinal(crd.Spec.Ordinal.Start).Build()
+		pod, err := NewPodBuilder(crd).WithOrdinal(crd.Spec.Ordinals.Start).Build()
 		require.NoError(t, err)
 		require.Equal(t, pod.Spec, pods[0].Object().Spec)
 	})
@@ -83,7 +83,7 @@ func TestBuildPods(t *testing.T) {
 					"agoric-6":  {DisableStrategy: ptr(cosmosv1.DisableAll)},
 					overridePod: {Image: overrideImage},
 				},
-				Ordinal: cosmosv1.Ordinal{
+				Ordinals: cosmosv1.Ordinals{
 					Start: 2,
 				},
 			},
@@ -115,7 +115,7 @@ func TestBuildPods(t *testing.T) {
 			},
 			Spec: cosmosv1.FullNodeSpec{
 				Replicas: 6,
-				Ordinal:  cosmosv1.Ordinal{Start: 2},
+				Ordinals: cosmosv1.Ordinals{Start: 2},
 			},
 			Status: cosmosv1.FullNodeStatus{
 				ScheduledSnapshotStatus: map[string]cosmosv1.FullNodeSnapshotStatus{
diff --git a/internal/fullnode/configmap_builder.go b/internal/fullnode/configmap_builder.go
index 45aa0f0e..598dd8bd 100644
--- a/internal/fullnode/configmap_builder.go
+++ b/internal/fullnode/configmap_builder.go
@@ -30,7 +30,7 @@ func BuildConfigMaps(crd *cosmosv1.CosmosFullNode, peers Peers) ([]diff.Resource
 	)
 	defer bufPool.Put(buf)
 	defer buf.Reset()
-	startOrdinal := crd.Spec.Ordinal.Start
+	startOrdinal := crd.Spec.Ordinals.Start
 
 	for i := startOrdinal; i < startOrdinal+crd.Spec.Replicas; i++ {
 		data := make(map[string]string)
diff --git a/internal/fullnode/node_key_builder.go b/internal/fullnode/node_key_builder.go
index bc2a8caf..c5bfa285 100644
--- a/internal/fullnode/node_key_builder.go
+++ b/internal/fullnode/node_key_builder.go
@@ -21,7 +21,7 @@ const nodeKeyFile = "node_key.json"
 // Returns an error if a new node key cannot be serialized. (Should never happen.)
 func BuildNodeKeySecrets(existing []*corev1.Secret, crd *cosmosv1.CosmosFullNode) ([]diff.Resource[*corev1.Secret], error) {
 	secrets := make([]diff.Resource[*corev1.Secret], 0, crd.Spec.Replicas)
-	startOrdinal := crd.Spec.Ordinal.Start
+	startOrdinal := crd.Spec.Ordinals.Start
 
 	for i := startOrdinal; i < startOrdinal+crd.Spec.Replicas; i++ {
 		var s corev1.Secret
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 4247dba2..57e32ee2 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -105,7 +105,7 @@ func NewPeerCollector(client Getter) *PeerCollector {
 // Collect peer information given the crd.
 func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode) (Peers, kube.ReconcileError) {
 	peers := make(Peers)
-	startOrdinal := crd.Spec.Ordinal.Start
+	startOrdinal := crd.Spec.Ordinals.Start
 
 	clusterDomain := "cluster.local"
 	if crd.Spec.Service.ClusterDomain != nil {
diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index 226f2210..f3188295 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -40,7 +40,7 @@ func BuildPVCs(
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		ordinal := i + crd.Spec.Ordinal.Start
+		ordinal := i + crd.Spec.Ordinals.Start
 		if pvcDisabled(crd, ordinal) {
 			continue
 		}
diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 7ba8c058..4cc88280 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -52,7 +52,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
 		for i := int32(0); i < crd.Spec.Replicas; i++ {
-			ordinal := i + crd.Spec.Ordinal.Start
+			ordinal := i + crd.Spec.Ordinals.Start
 			name := pvcName(crd, ordinal)
 			found := false
 			for _, pvc := range currentPVCs {
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 0c9aad32..054e582f 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -31,7 +31,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	}
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
-	startOrdinal := crd.Spec.Ordinal.Start
+	startOrdinal := crd.Spec.Ordinals.Start
 
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		ordinal := startOrdinal + i

From 212fd65fd7ac7b354b52999dce39ef4c293482d7 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Tue, 3 Dec 2024 09:40:42 -0700
Subject: [PATCH 29/41] Update autogenerated yaml file with 'ordinals'

---
 config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index 3dc55a10..95d8c470 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -555,7 +555,7 @@ spec:
                   Example: cosmos-1
                   Used for debugging.
                 type: object
-              ordinal:
+              ordinals:
                 description: Ordinal specifies the configuration for pod ordinal numbers
                 properties:
                   start:

From 8a24b650406cf81bbdaa24bc6ec1c28b2103ab00 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:06:02 -0700
Subject: [PATCH 30/41] match the description for start with that of the
 stateful set

---
 api/v1/cosmosfullnode_types.go                         |  6 +++++-
 .../crd/bases/cosmos.strange.love_cosmosfullnodes.yaml | 10 +++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 9b869af6..aee48085 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -34,7 +34,11 @@ const CosmosFullNodeController = "CosmosFullNode"
 
 // Ordinal specifies the configuration for pod ordinal numbers
 type Ordinals struct {
-	// Start specifies the initial ordinal number for pod naming
+	// start is the number representing the first replica's index. It may be used to number replicas from an alternate index (eg: 1-indexed) over the default 0-indexed names,
+	// or to orchestrate progressive movement of replicas from one CosmosFullnode spec to another. If set, replica indices will be in the range:
+	// [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+	// If unset, defaults to 0. Replica indices will be in the range:
+	// [0, .spec.replicas).
 	// +kubebuilder:validation:Minimum:=0
 	Start int32 `json:"start,omitempty"`
 }
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index 95d8c470..1dbcebd5 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -555,12 +555,16 @@ spec:
                   Example: cosmos-1
                   Used for debugging.
                 type: object
-              ordinals:
+              ordinal:
                 description: Ordinal specifies the configuration for pod ordinal numbers
                 properties:
                   start:
-                    description: Start specifies the initial ordinal number for pod
-                      naming
+                    description: |-
+                      start is the number representing the first replica's index. It may be used to number replicas from an alternate index (eg: 1-indexed) over the default 0-indexed names,
+                      or to orchestrate progressive movement of replicas from one CosmosFullnode spec to another. If set, replica indices will be in the range:
+                      [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+                      If unset, defaults to 0. Replica indices will be in the range:
+                      [0, .spec.replicas).
                     format: int32
                     minimum: 0
                     type: integer

From b1163e002a891868b54b14851a8cb9fe49fa0ea9 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:44:17 -0700
Subject: [PATCH 31/41] add happy path test with a non 0 starting ordinal

---
 internal/fullnode/configmap_builder_test.go | 58 +++++++++++++++++++--
 1 file changed, 53 insertions(+), 5 deletions(-)

diff --git a/internal/fullnode/configmap_builder_test.go b/internal/fullnode/configmap_builder_test.go
index 56dac97b..ef3a0e0f 100644
--- a/internal/fullnode/configmap_builder_test.go
+++ b/internal/fullnode/configmap_builder_test.go
@@ -42,16 +42,17 @@ func TestBuildConfigMaps(t *testing.T) {
 		crd.Namespace = "test"
 		crd.Spec.PodTemplate.Image = "agoric:v6.0.0"
 		crd.Spec.ChainSpec.Network = "testnet"
+		crd.Spec.Ordinals.Start = 0
 
 		cms, err := BuildConfigMaps(&crd, nil)
 		require.NoError(t, err)
-		require.Equal(t, 3, len(cms))
+		require.Equal(t, crd.Spec.Replicas, int32(len(cms)))
 
-		require.Equal(t, int64(0), cms[0].Ordinal())
+		require.Equal(t, crd.Spec.Ordinals.Start, int32(cms[0].Ordinal())+crd.Spec.Ordinals.Start)
 		require.NotEmpty(t, cms[0].Revision())
 
 		cm := cms[0].Object()
-		require.Equal(t, "agoric-0", cm.Name)
+		require.Equal(t, fmt.Sprintf("agoric-%d", crd.Spec.Ordinals.Start), cm.Name)
 		require.Equal(t, "test", cm.Namespace)
 		require.Nil(t, cm.Immutable)
 
@@ -59,7 +60,7 @@ func TestBuildConfigMaps(t *testing.T) {
 			"app.kubernetes.io/created-by": "cosmos-operator",
 			"app.kubernetes.io/component":  "CosmosFullNode",
 			"app.kubernetes.io/name":       "agoric",
-			"app.kubernetes.io/instance":   "agoric-0",
+			"app.kubernetes.io/instance":   fmt.Sprintf("%s-%d", crd.Name, crd.Spec.Ordinals.Start),
 			"app.kubernetes.io/version":    "v6.0.0",
 			"cosmos.strange.love/network":  "testnet",
 			"cosmos.strange.love/type":     "FullNode",
@@ -69,7 +70,54 @@ func TestBuildConfigMaps(t *testing.T) {
 		require.Equal(t, wantLabels, cm.Labels)
 
 		cm = cms[1].Object()
-		require.Equal(t, "agoric-1", cm.Name)
+		require.Equal(t, fmt.Sprintf("%s-%d", crd.Name, crd.Spec.Ordinals.Start+1), cm.Name)
+
+		require.NotEmpty(t, cms[0].Object().Data)
+		require.Equal(t, cms[0].Object().Data, cms[1].Object().Data)
+
+		crd.Spec.Type = cosmosv1.FullNode
+		cms2, err := BuildConfigMaps(&crd, nil)
+
+		require.NoError(t, err)
+		require.Equal(t, cms, cms2)
+	})
+
+	t.Run("happy path with non 0 starting ordinal", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Spec.Replicas = 3
+		crd.Name = "agoric"
+		crd.Namespace = "test"
+		crd.Spec.PodTemplate.Image = "agoric:v6.0.0"
+		crd.Spec.ChainSpec.Network = "testnet"
+		crd.Spec.Ordinals.Start = 2
+
+		cms, err := BuildConfigMaps(&crd, nil)
+		require.NoError(t, err)
+		require.Equal(t, crd.Spec.Replicas, int32(len(cms)))
+
+		require.Equal(t, crd.Spec.Ordinals.Start, int32(cms[0].Ordinal())+crd.Spec.Ordinals.Start)
+		require.NotEmpty(t, cms[0].Revision())
+
+		cm := cms[0].Object()
+		require.Equal(t, fmt.Sprintf("agoric-%d", crd.Spec.Ordinals.Start), cm.Name)
+		require.Equal(t, "test", cm.Namespace)
+		require.Nil(t, cm.Immutable)
+
+		wantLabels := map[string]string{
+			"app.kubernetes.io/created-by": "cosmos-operator",
+			"app.kubernetes.io/component":  "CosmosFullNode",
+			"app.kubernetes.io/name":       "agoric",
+			"app.kubernetes.io/instance":   fmt.Sprintf("%s-%d", crd.Name, crd.Spec.Ordinals.Start),
+			"app.kubernetes.io/version":    "v6.0.0",
+			"cosmos.strange.love/network":  "testnet",
+			"cosmos.strange.love/type":     "FullNode",
+		}
+		require.Empty(t, cm.Annotations)
+
+		require.Equal(t, wantLabels, cm.Labels)
+
+		cm = cms[1].Object()
+		require.Equal(t, fmt.Sprintf("%s-%d", crd.Name, crd.Spec.Ordinals.Start+1), cm.Name)
 
 		require.NotEmpty(t, cms[0].Object().Data)
 		require.Equal(t, cms[0].Object().Data, cms[1].Object().Data)

From f27d4a3737f834796e4eb87ab54c5b673f95d5e8 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:51:36 -0700
Subject: [PATCH 32/41] add node keybuilder test

---
 internal/fullnode/configmap_builder_test.go |  2 +-
 internal/fullnode/node_key_builder_test.go  | 52 +++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/internal/fullnode/configmap_builder_test.go b/internal/fullnode/configmap_builder_test.go
index ef3a0e0f..0158e757 100644
--- a/internal/fullnode/configmap_builder_test.go
+++ b/internal/fullnode/configmap_builder_test.go
@@ -42,7 +42,7 @@ func TestBuildConfigMaps(t *testing.T) {
 		crd.Namespace = "test"
 		crd.Spec.PodTemplate.Image = "agoric:v6.0.0"
 		crd.Spec.ChainSpec.Network = "testnet"
-		crd.Spec.Ordinals.Start = 0
+		//Default starting ordinal is 0
 
 		cms, err := BuildConfigMaps(&crd, nil)
 		require.NoError(t, err)
diff --git a/internal/fullnode/node_key_builder_test.go b/internal/fullnode/node_key_builder_test.go
index 6aad78f5..65019c4d 100644
--- a/internal/fullnode/node_key_builder_test.go
+++ b/internal/fullnode/node_key_builder_test.go
@@ -63,6 +63,58 @@ func TestBuildNodeKeySecrets(t *testing.T) {
 		}
 	})
 
+	t.Run("happy path with non 0 starting ordinal", func(t *testing.T) {
+		var crd cosmosv1.CosmosFullNode
+		crd.Namespace = "test-namespace"
+		crd.Name = "juno"
+		crd.Spec.Replicas = 3
+		crd.Spec.ChainSpec.Network = "mainnet"
+		crd.Spec.PodTemplate.Image = "ghcr.io/juno:v1.2.3"
+		// Start ordinal is 0 by default
+		crd.Spec.Ordinals.Start = 2
+
+		secrets, err := BuildNodeKeySecrets(nil, &crd)
+		require.NoError(t, err)
+		require.Equal(t, crd.Spec.Replicas, int32(len(secrets)))
+
+		for i, s := range secrets {
+			ordinal := crd.Spec.Ordinals.Start + int32(i)
+			require.Equal(t, crd.Spec.Ordinals.Start, crd.Spec.Ordinals.Start)
+			require.NotEmpty(t, s.Revision())
+
+			got := s.Object()
+			require.Equal(t, crd.Namespace, got.Namespace)
+			require.Equal(t, fmt.Sprintf("juno-node-key-%d", ordinal), got.Name)
+			require.Equal(t, "Secret", got.Kind)
+			require.Equal(t, "v1", got.APIVersion)
+
+			wantLabels := map[string]string{
+				"app.kubernetes.io/created-by": "cosmos-operator",
+				"app.kubernetes.io/component":  "CosmosFullNode",
+				"app.kubernetes.io/name":       "juno",
+				"app.kubernetes.io/instance":   fmt.Sprintf("%s-%d", crd.Name, ordinal),
+				"app.kubernetes.io/version":    "v1.2.3",
+				"cosmos.strange.love/network":  "mainnet",
+				"cosmos.strange.love/type":     "FullNode",
+			}
+			require.Equal(t, wantLabels, got.Labels)
+
+			require.Empty(t, got.Annotations)
+
+			require.True(t, *got.Immutable)
+			require.Equal(t, corev1.SecretTypeOpaque, got.Type)
+
+			nodeKey := got.Data["node_key.json"]
+			require.NotEmpty(t, nodeKey)
+
+			var gotJSON map[string]map[string]string
+			err = json.Unmarshal(nodeKey, &gotJSON)
+			require.NoError(t, err)
+			require.Equal(t, gotJSON["priv_key"]["type"], "tendermint/PrivKeyEd25519")
+			require.NotEmpty(t, gotJSON["priv_key"]["value"])
+		}
+	})
+
 	t.Run("with existing", func(t *testing.T) {
 		const namespace = "test-namespace"
 		var crd cosmosv1.CosmosFullNode

From 94afe93e7f07283bfb9618a635940494aedc4c79 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 12:57:37 -0700
Subject: [PATCH 33/41] test ofr peer collector

---
 internal/fullnode/peer_collector_test.go | 121 +++++++++++++++++++++++
 1 file changed, 121 insertions(+)

diff --git a/internal/fullnode/peer_collector_test.go b/internal/fullnode/peer_collector_test.go
index 8a379dc0..73b19dee 100644
--- a/internal/fullnode/peer_collector_test.go
+++ b/internal/fullnode/peer_collector_test.go
@@ -3,6 +3,7 @@ package fullnode
 import (
 	"context"
 	"errors"
+	"fmt"
 	"testing"
 
 	cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
@@ -148,6 +149,126 @@ func TestPeerCollector_Collect(t *testing.T) {
 		require.ElementsMatch(t, want, peers.AllExternal())
 	})
 
+	t.Run("happy path  with non 0 starting ordinal- private addresses", func(t *testing.T) {
+		var crd cosmosv1.CosmosFullNode
+		crd.Name = "dydx"
+		crd.Namespace = namespace
+		crd.Spec.Replicas = 2
+		crd.Spec.Ordinals.Start = 2
+
+		res, err := BuildNodeKeySecrets(nil, &crd)
+		require.NoError(t, err)
+		require.Equal(t, crd.Spec.Replicas, int32(len(res)))
+		secret := res[0].Object()
+		secret.Data[nodeKeyFile] = []byte(nodeKey)
+
+		var (
+			getCount int
+			objKeys  []client.ObjectKey
+		)
+		getter := mockGetter(func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			objKeys = append(objKeys, key)
+			getCount++
+			switch ref := obj.(type) {
+			case *corev1.Secret:
+				*ref = *secret
+			case *corev1.Service:
+				*ref = corev1.Service{}
+			}
+			return nil
+		})
+
+		collector := NewPeerCollector(getter)
+		peers, err := collector.Collect(ctx, &crd)
+		require.NoError(t, err)
+		require.Len(t, peers, 2)
+
+		require.Equal(t, 4, getCount) // 2 secrets + 2 services
+
+		wantKeys := []client.ObjectKey{
+			{Name: fmt.Sprintf("dydx-node-key-%d", crd.Spec.Ordinals.Start), Namespace: namespace},
+			{Name: fmt.Sprintf("dydx-p2p-%d", crd.Spec.Ordinals.Start), Namespace: namespace},
+			{Name: fmt.Sprintf("dydx-node-key-%d", crd.Spec.Ordinals.Start+1), Namespace: namespace},
+			{Name: fmt.Sprintf("dydx-p2p-%d", crd.Spec.Ordinals.Start+1), Namespace: namespace},
+		}
+		require.Equal(t, wantKeys, objKeys)
+
+		got := peers[client.ObjectKey{Name: fmt.Sprintf("dydx-%d", crd.Spec.Ordinals.Start), Namespace: namespace}]
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892", got.NodeID)
+		require.Equal(t, fmt.Sprintf("dydx-p2p-%d.strangelove.svc.cluster.local:26656", crd.Spec.Ordinals.Start), got.PrivateAddress)
+		require.Equal(t, fmt.Sprintf("1e23ce0b20ae2377925537cc71d1529d723bb892@dydx-p2p-%d.strangelove.svc.cluster.local:26656", crd.Spec.Ordinals.Start), got.PrivatePeer())
+		require.Empty(t, got.ExternalAddress)
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892@0.0.0.0:26656", got.ExternalPeer())
+
+		got = peers[client.ObjectKey{Name: fmt.Sprintf("dydx-%d", crd.Spec.Ordinals.Start+1), Namespace: namespace}]
+		require.NotEmpty(t, got.NodeID)
+		require.Equal(t, fmt.Sprintf("dydx-p2p-%d.strangelove.svc.cluster.local:26656", crd.Spec.Ordinals.Start+1), got.PrivateAddress)
+		require.Empty(t, got.ExternalAddress)
+
+		require.False(t, peers.HasIncompleteExternalAddress())
+	})
+
+	t.Run("happy path with non 0 starting ordinal - external addresses", func(t *testing.T) {
+		var crd cosmosv1.CosmosFullNode
+		crd.Name = "dydx"
+		crd.Namespace = namespace
+		crd.Spec.Replicas = 3
+		crd.Spec.Ordinals.Start = 0
+
+		res, err := BuildNodeKeySecrets(nil, &crd)
+		require.NoError(t, err)
+		require.Equal(t, crd.Spec.Replicas, int32(len(res)))
+		secret := res[0].Object()
+		secret.Data[nodeKeyFile] = []byte(nodeKey)
+
+		getter := mockGetter(func(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+			switch ref := obj.(type) {
+			case *corev1.Secret:
+				*ref = *secret
+			case *corev1.Service:
+				var svc corev1.Service
+				switch key.Name {
+				case fmt.Sprintf("dydx-p2p-%d", crd.Spec.Ordinals.Start):
+					svc.Spec.Type = corev1.ServiceTypeLoadBalancer
+				case fmt.Sprintf("dydx-p2p-%d", crd.Spec.Ordinals.Start+1):
+					svc.Spec.Type = corev1.ServiceTypeLoadBalancer
+					svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{{IP: "1.2.3.4"}}
+				case fmt.Sprintf("dydx-p2p-%d", crd.Spec.Ordinals.Start+2):
+					svc.Spec.Type = corev1.ServiceTypeLoadBalancer
+					svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{{Hostname: "host.example.com"}}
+				}
+				*ref = svc
+			}
+			return nil
+		})
+
+		collector := NewPeerCollector(getter)
+		peers, err := collector.Collect(ctx, &crd)
+		require.NoError(t, err)
+		require.Len(t, peers, 3)
+
+		got := peers[client.ObjectKey{Name: fmt.Sprintf("dydx-%d", crd.Spec.Ordinals.Start), Namespace: namespace}]
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892", got.NodeID)
+		require.Empty(t, got.ExternalAddress)
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892@0.0.0.0:26656", got.ExternalPeer())
+
+		got = peers[client.ObjectKey{Name: fmt.Sprintf("dydx-%d", crd.Spec.Ordinals.Start+1), Namespace: namespace}]
+		require.Equal(t, "1.2.3.4:26656", got.ExternalAddress)
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892@1.2.3.4:26656", got.ExternalPeer())
+
+		got = peers[client.ObjectKey{Name: fmt.Sprintf("dydx-%d", crd.Spec.Ordinals.Start+2), Namespace: namespace}]
+		require.Equal(t, "host.example.com:26656", got.ExternalAddress)
+		require.Equal(t, "1e23ce0b20ae2377925537cc71d1529d723bb892@host.example.com:26656", got.ExternalPeer())
+
+		require.True(t, peers.HasIncompleteExternalAddress())
+		want := []string{
+			"1e23ce0b20ae2377925537cc71d1529d723bb892@0.0.0.0:26656",
+			"1e23ce0b20ae2377925537cc71d1529d723bb892@1.2.3.4:26656",
+			"1e23ce0b20ae2377925537cc71d1529d723bb892@host.example.com:26656",
+		}
+		require.ElementsMatch(t, want, peers.AllExternal())
+	})
+
 	t.Run("zero replicas", func(t *testing.T) {
 		var crd cosmosv1.CosmosFullNode
 		crd.Spec.Replicas = 0

From 57019ed33c26ab9915a8565371b59dd48cd4ad94 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 13:07:12 -0700
Subject: [PATCH 34/41] pvc and service builder tests

---
 internal/fullnode/pvc_builder_test.go     | 62 +++++++++++++++++++++++
 internal/fullnode/service_builder_test.go | 52 +++++++++++++++++++
 2 files changed, 114 insertions(+)

diff --git a/internal/fullnode/pvc_builder_test.go b/internal/fullnode/pvc_builder_test.go
index dca1390b..b85950f9 100644
--- a/internal/fullnode/pvc_builder_test.go
+++ b/internal/fullnode/pvc_builder_test.go
@@ -72,6 +72,68 @@ func TestBuildPVCs(t *testing.T) {
 		}
 	})
 
+	t.Run("happy path with non 0 starting ordinal", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Name = "juno"
+		crd.Spec.Replicas = 3
+		crd.Spec.VolumeClaimTemplate.StorageClassName = "test-storage-class"
+		crd.Spec.Ordinals.Start = 2
+
+		crd.Spec.InstanceOverrides = map[string]cosmosv1.InstanceOverridesSpec{
+			fmt.Sprintf("juno-%d", crd.Spec.Ordinals.Start): {},
+		}
+
+		initial := BuildPVCs(&crd, map[int32]*dataSource{}, nil)
+		require.Equal(t, crd.Spec.Replicas, int32(len(initial)))
+		for _, r := range initial {
+			require.Equal(t, crd.Spec.Ordinals.Start, crd.Spec.Ordinals.Start)
+			require.NotEmpty(t, r.Revision())
+		}
+
+		initialPVCs := lo.Map(initial, func(r diff.Resource[*corev1.PersistentVolumeClaim], _ int) *corev1.PersistentVolumeClaim {
+			return r.Object()
+		})
+
+		pvcs := lo.Map(BuildPVCs(&crd, map[int32]*dataSource{}, initialPVCs), func(r diff.Resource[*corev1.PersistentVolumeClaim], _ int) *corev1.PersistentVolumeClaim {
+			return r.Object()
+		})
+
+		require.Equal(t, crd.Spec.Replicas, int32(len(pvcs)))
+
+		wantNames := make([]string, crd.Spec.Replicas)
+		for i := range wantNames {
+			wantNames[i] = fmt.Sprintf("pvc-juno-%d", crd.Spec.Ordinals.Start+int32(i))
+		}
+
+		gotNames := lo.Map(pvcs, func(pvc *corev1.PersistentVolumeClaim, _ int) string { return pvc.Name })
+		require.Equal(t, wantNames, gotNames)
+
+		for i, got := range pvcs {
+			ordinal := crd.Spec.Ordinals.Start + int32(i)
+			require.Equal(t, crd.Namespace, got.Namespace)
+			require.Equal(t, "PersistentVolumeClaim", got.Kind)
+			require.Equal(t, "v1", got.APIVersion)
+
+			wantLabels := map[string]string{
+				"app.kubernetes.io/created-by": "cosmos-operator",
+				"app.kubernetes.io/component":  "CosmosFullNode",
+				"app.kubernetes.io/name":       "juno",
+				"app.kubernetes.io/instance":   fmt.Sprintf("juno-%d", ordinal),
+				"app.kubernetes.io/version":    "v1.2.3",
+				"cosmos.strange.love/network":  "mainnet",
+				"cosmos.strange.love/type":     "FullNode",
+			}
+			require.Equal(t, wantLabels, got.Labels)
+
+			require.Len(t, got.Spec.AccessModes, 1)
+			require.Equal(t, corev1.ReadWriteOnce, got.Spec.AccessModes[0])
+
+			require.Equal(t, crd.Spec.VolumeClaimTemplate.Resources, got.Spec.Resources)
+			require.Equal(t, "test-storage-class", *got.Spec.StorageClassName)
+			require.Equal(t, corev1.PersistentVolumeFilesystem, *got.Spec.VolumeMode)
+		}
+	})
+
 	t.Run("advanced configuration", func(t *testing.T) {
 		crd := defaultCRD()
 		crd.Spec.Replicas = 1
diff --git a/internal/fullnode/service_builder_test.go b/internal/fullnode/service_builder_test.go
index 246af0fc..571b8e66 100644
--- a/internal/fullnode/service_builder_test.go
+++ b/internal/fullnode/service_builder_test.go
@@ -68,6 +68,58 @@ func TestBuildServices(t *testing.T) {
 		}
 	})
 
+	t.Run("p2p services", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Spec.Replicas = 3
+		crd.Name = "terra"
+		crd.Namespace = "test"
+		crd.Spec.ChainSpec.Network = "testnet"
+		crd.Spec.PodTemplate.Image = "terra:v6.0.0"
+		crd.Spec.Ordinals.Start = 2
+
+		svcs := BuildServices(&crd)
+
+		require.Equal(t, 4, len(svcs)) // 3 p2p services + 1 rpc service
+
+		for i := 0; i < int(crd.Spec.Replicas); i++ {
+			ordinal := crd.Spec.Ordinals.Start + int32(i)
+			p2p := svcs[i].Object()
+			require.Equal(t, fmt.Sprintf("terra-p2p-%d", ordinal), p2p.Name)
+			require.Equal(t, "test", p2p.Namespace)
+
+			wantLabels := map[string]string{
+				"app.kubernetes.io/created-by": "cosmos-operator",
+				"app.kubernetes.io/name":       "terra",
+				"app.kubernetes.io/component":  "p2p",
+				"app.kubernetes.io/version":    "v6.0.0",
+				"app.kubernetes.io/instance":   fmt.Sprintf("terra-%d", ordinal),
+				"cosmos.strange.love/network":  "testnet",
+				"cosmos.strange.love/type":     "FullNode",
+			}
+			require.Equal(t, wantLabels, p2p.Labels)
+
+			wantSpec := corev1.ServiceSpec{
+				Ports: []corev1.ServicePort{
+					{
+						Name:       "p2p",
+						Protocol:   corev1.ProtocolTCP,
+						Port:       26656,
+						TargetPort: intstr.FromString("p2p"),
+					},
+				},
+				Selector: map[string]string{"app.kubernetes.io/instance": fmt.Sprintf("terra-%d", ordinal)},
+				Type:     corev1.ServiceTypeClusterIP,
+			}
+			// By default, expose the first p2p service publicly.
+			if i == 0 {
+				wantSpec.Type = corev1.ServiceTypeLoadBalancer
+				wantSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal
+			}
+
+			require.Equal(t, wantSpec, p2p.Spec)
+		}
+	})
+
 	t.Run("p2p max external addresses", func(t *testing.T) {
 		crd := defaultCRD()
 		crd.Spec.Replicas = 3

From e0afa9b60cf4025349dddbf7b7d23c4b01171331 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 13:17:12 -0700
Subject: [PATCH 35/41] Ensure correct tags for goloang json

---
 api/v1/cosmosfullnode_types.go                            | 2 +-
 config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index aee48085..67fcb10d 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -49,7 +49,7 @@ type FullNodeSpec struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	// Ordinal specifies the configuration for pod ordinal numbers
-	Ordinals Ordinals `json:"ordinal,omitempty"`
+	Ordinals Ordinals `json:"ordinals,omitempty"`
 
 	// Number of replicas to create.
 	// Individual replicas have a consistent identity.
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index 1dbcebd5..f780a3f7 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -555,7 +555,7 @@ spec:
                   Example: cosmos-1
                   Used for debugging.
                 type: object
-              ordinal:
+              ordinals:
                 description: Ordinal specifies the configuration for pod ordinal numbers
                 properties:
                   start:

From 45951395022774501b7d299852a2b0f966627ce9 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 13:40:28 -0700
Subject: [PATCH 36/41] make for loops consistent across all builders

---
 internal/fullnode/pvc_builder.go     | 10 +++++-----
 internal/fullnode/pvc_control.go     |  7 +++----
 internal/fullnode/service_builder.go | 11 ++++-------
 3 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index f3188295..b59ad55d 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -39,16 +39,16 @@ func BuildPVCs(
 	}
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		ordinal := i + crd.Spec.Ordinals.Start
-		if pvcDisabled(crd, ordinal) {
+	for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
+
+		if pvcDisabled(crd, i) {
 			continue
 		}
 
 		pvc := base.DeepCopy()
-		name := pvcName(crd, ordinal)
+		name := pvcName(crd, i)
 		pvc.Name = name
-		podName := instanceName(crd, ordinal)
+		podName := instanceName(crd, i)
 		pvc.Labels[kube.InstanceLabel] = podName
 
 		var dataSource *corev1.TypedLocalObjectReference
diff --git a/internal/fullnode/pvc_control.go b/internal/fullnode/pvc_control.go
index 4cc88280..5d87b693 100644
--- a/internal/fullnode/pvc_control.go
+++ b/internal/fullnode/pvc_control.go
@@ -51,9 +51,8 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 
 	dataSources := make(map[int32]*dataSource)
 	if len(currentPVCs) < int(crd.Spec.Replicas) {
-		for i := int32(0); i < crd.Spec.Replicas; i++ {
-			ordinal := i + crd.Spec.Ordinals.Start
-			name := pvcName(crd, ordinal)
+		for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
+			name := pvcName(crd, i)
 			found := false
 			for _, pvc := range currentPVCs {
 				if pvc.Name == name {
@@ -62,7 +61,7 @@ func (control PVCControl) Reconcile(ctx context.Context, reporter kube.Reporter,
 				}
 			}
 			if !found {
-				ds := control.findDataSource(ctx, reporter, crd, ordinal)
+				ds := control.findDataSource(ctx, reporter, crd, i)
 				if ds == nil {
 					ds = &dataSource{
 						size: crd.Spec.VolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage],
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 054e582f..d2c78aac 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -31,18 +31,15 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	}
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
-	startOrdinal := crd.Spec.Ordinals.Start
-
-	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		ordinal := startOrdinal + i
+	for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
 		var svc corev1.Service
-		svc.Name = p2pServiceName(crd, ordinal)
+		svc.Name = p2pServiceName(crd, i)
 		svc.Namespace = crd.Namespace
 		svc.Kind = "Service"
 		svc.APIVersion = "v1"
 
 		svc.Labels = defaultLabels(crd,
-			kube.InstanceLabel, instanceName(crd, ordinal),
+			kube.InstanceLabel, instanceName(crd, i),
 			kube.ComponentLabel, "p2p",
 		)
 		svc.Annotations = map[string]string{}
@@ -55,7 +52,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 				TargetPort: intstr.FromString("p2p"),
 			},
 		}
-		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, ordinal)}
+		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, i)}
 
 		if i < maxExternal {
 			preserveMergeInto(svc.Labels, crd.Spec.Service.P2PTemplate.Metadata.Labels)

From 79e5c854d4a1c015b2222208c5fe5776df27733f Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 14:11:41 -0700
Subject: [PATCH 37/41] fix service builder loop. seperate index from ordinal
 name since both are used

---
 internal/fullnode/service_builder.go | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index d2c78aac..476d23c5 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -31,19 +31,21 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	}
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
-	for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
+	startOrdinal := crd.Spec.Ordinals.Start
+
+	for i := int32(0); i < crd.Spec.Replicas; i++ {
+		ordinal := startOrdinal + i
 		var svc corev1.Service
-		svc.Name = p2pServiceName(crd, i)
+		svc.Name = p2pServiceName(crd, ordinal)
 		svc.Namespace = crd.Namespace
 		svc.Kind = "Service"
 		svc.APIVersion = "v1"
 
 		svc.Labels = defaultLabels(crd,
-			kube.InstanceLabel, instanceName(crd, i),
+			kube.InstanceLabel, instanceName(crd, ordinal),
 			kube.ComponentLabel, "p2p",
 		)
 		svc.Annotations = map[string]string{}
-
 		svc.Spec.Ports = []corev1.ServicePort{
 			{
 				Name:       "p2p",
@@ -52,7 +54,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 				TargetPort: intstr.FromString("p2p"),
 			},
 		}
-		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, i)}
+		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, ordinal)}
 
 		if i < maxExternal {
 			preserveMergeInto(svc.Labels, crd.Spec.Service.P2PTemplate.Metadata.Labels)
@@ -63,15 +65,11 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			svc.Spec.Type = corev1.ServiceTypeClusterIP
 			svc.Spec.ClusterIP = *valOrDefault(crd.Spec.Service.P2PTemplate.ClusterIP, ptr(""))
 		}
-
 		p2ps[i] = diff.Adapt(&svc, int(i))
 	}
-
 	rpc := rpcService(crd)
-
 	return append(p2ps, diff.Adapt(rpc, len(p2ps)))
 }
-
 func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	var svc corev1.Service
 	svc.Name = rpcServiceName(crd)
@@ -82,7 +80,6 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 		kube.ComponentLabel, "rpc",
 	)
 	svc.Annotations = map[string]string{}
-
 	svc.Spec.Ports = []corev1.ServicePort{
 		{
 			Name:       "api",
@@ -115,15 +112,12 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 			TargetPort: intstr.FromString("grpc-web"),
 		},
 	}
-
 	svc.Spec.Selector = map[string]string{kube.NameLabel: appName(crd)}
 	svc.Spec.Type = corev1.ServiceTypeClusterIP
-
 	rpcSpec := crd.Spec.Service.RPCTemplate
 	preserveMergeInto(svc.Labels, rpcSpec.Metadata.Labels)
 	preserveMergeInto(svc.Annotations, rpcSpec.Metadata.Annotations)
 	kube.NormalizeMetadata(&svc.ObjectMeta)
-
 	if v := rpcSpec.ExternalTrafficPolicy; v != nil {
 		svc.Spec.ExternalTrafficPolicy = *v
 	}
@@ -133,14 +127,11 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	if v := rpcSpec.ClusterIP; v != nil {
 		svc.Spec.ClusterIP = *v
 	}
-
 	return &svc
 }
-
 func p2pServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string {
 	return fmt.Sprintf("%s-p2p-%d", appName(crd), ordinal)
 }
-
 func rpcServiceName(crd *cosmosv1.CosmosFullNode) string {
 	return fmt.Sprintf("%s-rpc", appName(crd))
 }

From c8ba8cead4a0308b85d43ec52ba9dd4eeecfacd5 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 14:27:08 -0700
Subject: [PATCH 38/41] potential lint error fix (new lines between functions)

---
 internal/fullnode/service_builder.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 476d23c5..894ba176 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -31,16 +31,13 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	}
 	maxExternal := lo.Clamp(max, 0, crd.Spec.Replicas)
 	p2ps := make([]diff.Resource[*corev1.Service], crd.Spec.Replicas)
-	startOrdinal := crd.Spec.Ordinals.Start
-
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
-		ordinal := startOrdinal + i
+		ordinal := crd.Spec.Ordinals.Start + i
 		var svc corev1.Service
 		svc.Name = p2pServiceName(crd, ordinal)
 		svc.Namespace = crd.Namespace
 		svc.Kind = "Service"
 		svc.APIVersion = "v1"
-
 		svc.Labels = defaultLabels(crd,
 			kube.InstanceLabel, instanceName(crd, ordinal),
 			kube.ComponentLabel, "p2p",
@@ -55,7 +52,6 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			},
 		}
 		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, ordinal)}
-
 		if i < maxExternal {
 			preserveMergeInto(svc.Labels, crd.Spec.Service.P2PTemplate.Metadata.Labels)
 			preserveMergeInto(svc.Annotations, crd.Spec.Service.P2PTemplate.Metadata.Annotations)
@@ -70,6 +66,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 	rpc := rpcService(crd)
 	return append(p2ps, diff.Adapt(rpc, len(p2ps)))
 }
+
 func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	var svc corev1.Service
 	svc.Name = rpcServiceName(crd)
@@ -129,9 +126,11 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	}
 	return &svc
 }
+
 func p2pServiceName(crd *cosmosv1.CosmosFullNode, ordinal int32) string {
 	return fmt.Sprintf("%s-p2p-%d", appName(crd), ordinal)
 }
+
 func rpcServiceName(crd *cosmosv1.CosmosFullNode) string {
 	return fmt.Sprintf("%s-rpc", appName(crd))
 }

From 0bc73a8fc098dcc6698fab42057927536b70bcac Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Wed, 4 Dec 2024 14:32:51 -0700
Subject: [PATCH 39/41] remove unnecessary new line

---
 internal/fullnode/pvc_builder.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/fullnode/pvc_builder.go b/internal/fullnode/pvc_builder.go
index b59ad55d..588586d5 100644
--- a/internal/fullnode/pvc_builder.go
+++ b/internal/fullnode/pvc_builder.go
@@ -40,7 +40,6 @@ func BuildPVCs(
 
 	var pvcs []diff.Resource[*corev1.PersistentVolumeClaim]
 	for i := crd.Spec.Ordinals.Start; i < crd.Spec.Ordinals.Start+crd.Spec.Replicas; i++ {
-
 		if pvcDisabled(crd, i) {
 			continue
 		}

From 2749a796e2d3946881290bbe3d747bba7976c095 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Thu, 5 Dec 2024 00:37:55 -0700
Subject: [PATCH 40/41] Update the Ordinals description to match language
 similar to be similar to the Stateful Set description of Ordinals

---
 api/v1/cosmosfullnode_types.go                            | 4 ++--
 config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 67fcb10d..0c7ecab0 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -32,7 +32,6 @@ const CosmosFullNodeController = "CosmosFullNode"
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
-// Ordinal specifies the configuration for pod ordinal numbers
 type Ordinals struct {
 	// start is the number representing the first replica's index. It may be used to number replicas from an alternate index (eg: 1-indexed) over the default 0-indexed names,
 	// or to orchestrate progressive movement of replicas from one CosmosFullnode spec to another. If set, replica indices will be in the range:
@@ -48,7 +47,8 @@ type FullNodeSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// Ordinal specifies the configuration for pod ordinal numbers
+	// Ordinals controls the numbering of replica indices in a CosmosFullnode spec.
+	// The default ordinals behavior assigns a "0" index to the first replica and increments the index by one for each additional replica requested.
 	Ordinals Ordinals `json:"ordinals,omitempty"`
 
 	// Number of replicas to create.
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index f780a3f7..d20af7e6 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -556,7 +556,9 @@ spec:
                   Used for debugging.
                 type: object
               ordinals:
-                description: Ordinal specifies the configuration for pod ordinal numbers
+                description: |-
+                  Ordinals controls the numbering of replica indices in a CosmosFullnode spec.
+                  The default ordinals behavior assigns a "0" index to the first replica and increments the index by one for each additional replica requested.
                 properties:
                   start:
                     description: |-

From 1c27636f9f296c83096976892a3411611203eea8 Mon Sep 17 00:00:00 2001
From: vimystic <122659254+vimystic@users.noreply.github.com>
Date: Thu, 12 Dec 2024 16:18:08 -0700
Subject: [PATCH 41/41] update with no starting ordinal defined. i.e: Default
 value 0

---
 internal/fullnode/build_pods_test.go | 113 +++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)

diff --git a/internal/fullnode/build_pods_test.go b/internal/fullnode/build_pods_test.go
index 732b45af..a5f216d5 100644
--- a/internal/fullnode/build_pods_test.go
+++ b/internal/fullnode/build_pods_test.go
@@ -136,4 +136,117 @@ func TestBuildPods(t *testing.T) {
 		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
 		require.Equal(t, want, got)
 	})
+
+	t.Run("happy path without starting ordinal", func(t *testing.T) {
+		crd := &cosmosv1.CosmosFullNode{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "agoric",
+				Namespace: "test",
+			},
+			Spec: cosmosv1.FullNodeSpec{
+				Replicas:  5,
+				ChainSpec: cosmosv1.ChainSpec{Network: "devnet"},
+				PodTemplate: cosmosv1.PodSpec{
+					Image: "busybox:latest",
+				},
+				InstanceOverrides: nil,
+			},
+		}
+
+		cksums := make(ConfigChecksums)
+		for i := 0; i < int(crd.Spec.Replicas); i++ {
+			cksums[client.ObjectKey{Namespace: crd.Namespace, Name: fmt.Sprintf("agoric-%d", i+int(crd.Spec.Ordinals.Start))}] = strconv.Itoa(i + int(crd.Spec.Ordinals.Start))
+		}
+
+		pods, err := BuildPods(crd, cksums)
+		require.NoError(t, err)
+		require.Equal(t, 5, len(pods))
+
+		for i, r := range pods {
+			expectedOrdinal := crd.Spec.Ordinals.Start + int32(i)
+			require.Equal(t, int64(expectedOrdinal), r.Ordinal(), i)
+			require.NotEmpty(t, r.Revision(), i)
+			require.Equal(t, strconv.Itoa(int(expectedOrdinal)), r.Object().Annotations["cosmos.strange.love/config-checksum"])
+		}
+
+		want := lo.Map([]int{0, 1, 2, 3, 4}, func(i int, _ int) string {
+			return fmt.Sprintf("agoric-%d", i)
+		})
+		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
+		require.Equal(t, want, got)
+
+		pod, err := NewPodBuilder(crd).WithOrdinal(crd.Spec.Ordinals.Start).Build()
+		require.NoError(t, err)
+		require.Equal(t, pod.Spec, pods[0].Object().Spec)
+	})
+
+	t.Run("instance overrides without starting ordinal", func(t *testing.T) {
+		const (
+			image         = "agoric:latest"
+			overrideImage = "some_image:custom"
+			overridePod   = "agoric-5"
+		)
+		crd := &cosmosv1.CosmosFullNode{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "agoric",
+			},
+			Spec: cosmosv1.FullNodeSpec{
+				Replicas: 6,
+				PodTemplate: cosmosv1.PodSpec{
+					Image: image,
+				},
+				InstanceOverrides: map[string]cosmosv1.InstanceOverridesSpec{
+					"agoric-2":  {DisableStrategy: ptr(cosmosv1.DisablePod)},
+					"agoric-4":  {DisableStrategy: ptr(cosmosv1.DisableAll)},
+					overridePod: {Image: overrideImage},
+				},
+			},
+		}
+
+		pods, err := BuildPods(crd, nil)
+		require.NoError(t, err)
+		require.Equal(t, 4, len(pods))
+
+		want := lo.Map([]int{0, 1, 3, 5}, func(i int, _ int) string {
+			return fmt.Sprintf("agoric-%d", i)
+		})
+		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
+		require.Equal(t, want, got)
+		for _, pod := range pods {
+			image := pod.Object().Spec.Containers[0].Image
+			if pod.Object().Name == overridePod {
+				require.Equal(t, overrideImage, image)
+			} else {
+				require.Equal(t, image, image)
+			}
+		}
+	})
+
+	t.Run("scheduled volume snapshot pod candidate without starting ordinal", func(t *testing.T) {
+		crd := &cosmosv1.CosmosFullNode{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "agoric",
+			},
+			Spec: cosmosv1.FullNodeSpec{
+				Replicas: 6,
+			},
+			Status: cosmosv1.FullNodeStatus{
+				ScheduledSnapshotStatus: map[string]cosmosv1.FullNodeSnapshotStatus{
+					"some.scheduled.snapshot.1":       {PodCandidate: "agoric-1"},
+					"some.scheduled.snapshot.2":       {PodCandidate: "agoric-2"},
+					"some.scheduled.snapshot.ignored": {PodCandidate: "agoric-99"},
+				},
+			},
+		}
+
+		pods, err := BuildPods(crd, nil)
+		require.NoError(t, err)
+		require.Equal(t, 4, len(pods))
+
+		want := lo.Map([]int{0, 3, 4, 5}, func(i int, _ int) string {
+			return fmt.Sprintf("agoric-%d", i)
+		})
+		got := lo.Map(pods, func(pod diff.Resource[*corev1.Pod], _ int) string { return pod.Object().Name })
+		require.Equal(t, want, got)
+	})
 }