Initial implementation for Migrate DB On StartUp and Jobs Approach

api/v1alpha08/sonataflow_persistence_types.go

@@ -14,6 +14,14 @@
 
 package v1alpha08
 
+type DBMigrationStrategyType string
+
+const (
+	DBMigrationStrategyService DBMigrationStrategyType = "service"
+	DBMigrationStrategyJob     DBMigrationStrategyType = "job"
+	DBMigrationStrategyNone    DBMigrationStrategyType = "none"
+)
+
 // PlatformPersistenceOptionsSpec configures the DataBase in the platform spec. This specification can
 // be used by workflows and platform services when they don't provide one of their own.
 // +optional
@@ -51,10 +59,13 @@ type PersistenceOptionsSpec struct {
 	// +optional
 	PostgreSQL *PersistencePostgreSQL `json:"postgresql,omitempty"`
 
-	// Whether to migrate database on service startup?
+	// DB Migration approach for service?
+	// job: use job based approach
+	// service: service itself shall migrate the db
+	// none: no db migration needed
 	// +optional
-	// +default: false
-	MigrateDBOnStartUp bool `json:"migrateDBOnStartUp"`
+	// +kubebuilder:default:=service
+	DBMigrationStrategy string `json:"dbMigrationStrategy,omitempty"`
 }
 
 // PersistencePostgreSQL configure postgresql connection for service(s).

api/v1alpha08/sonataflowplatform_types.go

@@ -92,6 +92,31 @@ const (
 	PlatformDuplicatedReason = "Duplicated"
 )
 
+type DBMigrationStatus string
+
+const (
+	DBMigrationStatusStarted    DBMigrationStatus = "Started"
+	DBMigrationStatusInProgress DBMigrationStatus = "In-Progress"
+	DBMigrationStatusSucceeded  DBMigrationStatus = "Succeeded"
+	DBMigrationStatusFailed     DBMigrationStatus = "Failed"
+
+	MessageDBMigrationStatusStarted    string = "Started the database migrations for the services"
+	MessageDBMigrationStatusInProgress string = "The database migrations for the services are in-progress"
+	MessageDBMigrationStatusSucceeded  string = "The database migrations for the services are successful"
+	MessageDBMigrationStatusFailed     string = "The database migrations for the services have failed"
+
+	ReasonDBMigrationStatusStarted    string = "Started by SonataFlow operator"
+	ReasonDBMigrationStatusInProgress string = "The database migration job is in-progress"
+	ReasonDBMigrationStatusSucceeded  string = "The database migration job completed as expected"
+	ReasonDBMigrationStatusFailed     string = "The database may be unreachable, invalid credentials supplied or flyway migration failed. Please check logs for further details."
+)
+
+type SonataFlowPlatformDBMigrationStatus struct {
+	Status  DBMigrationStatus `json:"dbMigrationStatus,omitempty"`
+	Message string            `json:"message,omitempty"`
+	Reason  string            `json:"reason,omitempty"`
+}
+
 // SonataFlowPlatformStatus defines the observed state of SonataFlowPlatform
 // +k8s:openapi-gen=true
 type SonataFlowPlatformStatus struct {
@@ -111,6 +136,8 @@ type SonataFlowPlatformStatus struct {
 	// Triggers list of triggers created for the SonataFlowPlatform
 	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="triggers"
 	Triggers []SonataFlowPlatformTriggerRef `json:"triggers,omitempty"`
+	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="dbMigrationStatus"
+	SonataFlowPlatformDBMigrationStatus *SonataFlowPlatformDBMigrationStatus `json:"sonataFlowPlatformDBMigrationStatus,omitempty"`
 }
 
 // SonataFlowPlatformTriggerRef defines a trigger created for the SonataFlowPlatform.
@@ -121,6 +148,8 @@ type SonataFlowPlatformTriggerRef struct {
 	// Namespace of the Trigger
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_NS"
 	Namespace string `json:"namespace"`
+	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="dbMigrationStatus"
+	SonataFlowPlatformDBMigrationStatus *SonataFlowPlatformDBMigrationStatus `json:"sonataFlowPlatformDBMigrationStatus,omitempty"`
 }
 
 // SonataFlowClusterPlatformRefStatus information related to the (optional) active SonataFlowClusterPlatform

bundle/manifests/sonataflow.org_sonataflowplatforms.yaml

@@ -665,9 +665,14 @@ spec:
                           by default.
                         maxProperties: 2
                         properties:
-                          migrateDBOnStartUp:
-                            description: Whether to migrate database on service startup?
-                            type: boolean
+                          dbMigrationStrategy:
+                            default: service
+                            description: |-
+                              DB Migration approach for service?
+                              job: use job based approach
+                              service: service itself shall migrate the db
+                              none: no db migration needed
+                            type: string
                           postgresql:
                             description: Connect configured services to a postgresql
                               database.
@@ -8709,9 +8714,14 @@ spec:
                           by default.
                         maxProperties: 2
                         properties:
-                          migrateDBOnStartUp:
-                            description: Whether to migrate database on service startup?
-                            type: boolean
+                          dbMigrationStrategy:
+                            default: service
+                            description: |-
+                              DB Migration approach for service?
+                              job: use job based approach
+                              service: service itself shall migrate the db
+                              none: no db migration needed
+                            type: string
                           postgresql:
                             description: Connect configured services to a postgresql
                               database.
@@ -16886,6 +16896,15 @@ spec:
                 description: The generation observed by the deployment controller.
                 format: int64
                 type: integer
+              sonataFlowPlatformDBMigrationStatus:
+                properties:
+                  dbMigrationStatus:
+                    type: string
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                type: object
               triggers:
                 description: Triggers list of triggers created for the SonataFlowPlatform
                 items:
@@ -16898,6 +16917,15 @@ spec:
                     namespace:
                       description: Namespace of the Trigger
                       type: string
+                    sonataFlowPlatformDBMigrationStatus:
+                      properties:
+                        dbMigrationStatus:
+                          type: string
+                        message:
+                          type: string
+                        reason:
+                          type: string
+                      type: object
                   required:
                   - name
                   - namespace

bundle/manifests/sonataflow.org_sonataflows.yaml

@@ -1997,9 +1997,14 @@ spec:
                   for the workflow
                 maxProperties: 2
                 properties:
-                  migrateDBOnStartUp:
-                    description: Whether to migrate database on service startup?
-                    type: boolean
+                  dbMigrationStrategy:
+                    default: service
+                    description: |-
+                      DB Migration approach for service?
+                      job: use job based approach
+                      service: service itself shall migrate the db
+                      none: no db migration needed
+                    type: string
                   postgresql:
                     description: Connect configured services to a postgresql database.
                     maxProperties: 2

config/crd/bases/sonataflow.org_sonataflowplatforms.yaml

@@ -665,9 +665,14 @@ spec:
                           by default.
                         maxProperties: 2
                         properties:
-                          migrateDBOnStartUp:
-                            description: Whether to migrate database on service startup?
-                            type: boolean
+                          dbMigrationStrategy:
+                            default: service
+                            description: |-
+                              DB Migration approach for service?
+                              job: use job based approach
+                              service: service itself shall migrate the db
+                              none: no db migration needed
+                            type: string
                           postgresql:
                             description: Connect configured services to a postgresql
                               database.
@@ -8709,9 +8714,14 @@ spec:
                           by default.
                         maxProperties: 2
                         properties:
-                          migrateDBOnStartUp:
-                            description: Whether to migrate database on service startup?
-                            type: boolean
+                          dbMigrationStrategy:
+                            default: service
+                            description: |-
+                              DB Migration approach for service?
+                              job: use job based approach
+                              service: service itself shall migrate the db
+                              none: no db migration needed
+                            type: string
                           postgresql:
                             description: Connect configured services to a postgresql
                               database.
@@ -16886,6 +16896,15 @@ spec:
                 description: The generation observed by the deployment controller.
                 format: int64
                 type: integer
+              sonataFlowPlatformDBMigrationStatus:
+                properties:
+                  dbMigrationStatus:
+                    type: string
+                  message:
+                    type: string
+                  reason:
+                    type: string
+                type: object
               triggers:
                 description: Triggers list of triggers created for the SonataFlowPlatform
                 items:
@@ -16898,6 +16917,15 @@ spec:
                     namespace:
                       description: Namespace of the Trigger
                       type: string
+                    sonataFlowPlatformDBMigrationStatus:
+                      properties:
+                        dbMigrationStatus:
+                          type: string
+                        message:
+                          type: string
+                        reason:
+                          type: string
+                      type: object
                   required:
                   - name
                   - namespace

config/crd/bases/sonataflow.org_sonataflows.yaml

@@ -1997,9 +1997,14 @@ spec:
                   for the workflow
                 maxProperties: 2
                 properties:
-                  migrateDBOnStartUp:
-                    description: Whether to migrate database on service startup?
-                    type: boolean
+                  dbMigrationStrategy:
+                    default: service
+                    description: |-
+                      DB Migration approach for service?
+                      job: use job based approach
+                      service: service itself shall migrate the db
+                      none: no db migration needed
+                    type: string
                   postgresql:
                     description: Connect configured services to a postgresql database.
                     maxProperties: 2

config/manifests/bases/sonataflow-operator.clusterserviceversion.yaml

@@ -164,9 +164,13 @@ spec:
       - description: Info generic information related to the Platform
         displayName: info
         path: info
+      - displayName: dbMigrationStatus
+        path: sonataFlowPlatformDBMigrationStatus
       - description: Triggers list of triggers created for the SonataFlowPlatform
         displayName: triggers
         path: triggers
+      - displayName: dbMigrationStatus
+        path: triggers[0].sonataFlowPlatformDBMigrationStatus
       - description: Version the operator version controlling this Platform
         displayName: version
         path: version

config/rbac/role.yaml

@@ -4,6 +4,18 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - sonataflow.org
   resources:

images/tools/sonataflow-db-migrator/build-container-image.sh

@@ -33,7 +33,7 @@ cleanup () {
 # Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it.
 OPERATOR_VERSION=999.0.0 # comes from version.go
 DDL_VERSION=10.0.999-SNAPSHOT
-DDL_FILE=kogito-ddl-10.0.999-20240806.011718-23-db-scripts.zip
+DDL_FILE=kogito-ddl-10.0.999-20241002.021447-65-db-scripts.zip
 DDL_BASE_URL=https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl
 CEKIT_BUILDER=podman
 

internal/controller/cfg/controllers_cfg.go

@@ -31,6 +31,20 @@ const (
 	defaultConfigMountPath = "/config/controllers_cfg.yaml"
 )
 
+const (
+	dbMigrationJobName       = "sonataflow-db-migrator-job"
+	dbMigrationContainerName = "db-migration-container"
+	dbMigratorToolImage      = "quay.io/rhkp/incubator-kie-kogito-service-db-migration-postgresql:latest"
+	dbMigrationCmd           = "./migration.sh"
+)
+
+type DBMigrationJobCfg struct {
+	JobName       string `yaml:"jobName,omitempty"`
+	ContainerName string `yaml:"containerName,omitempty"`
+	ToolImageName string `yaml:"toolImageName,omitempty"`
+	MigrationCmd  string `yaml:"migrationCmd,omitempty"`
+}
+
 var controllersCfg *ControllersCfg
 
 var defaultControllersCfg = &ControllersCfg{
@@ -109,3 +123,12 @@ func GetCfg() *ControllersCfg {
 	}
 	return controllersCfg
 }
+
+func GetDBMigrationJobCfg() *DBMigrationJobCfg {
+	return &DBMigrationJobCfg{
+		JobName:       dbMigrationJobName,
+		ContainerName: dbMigrationContainerName,
+		ToolImageName: dbMigratorToolImage,
+		MigrationCmd:  dbMigrationCmd,
+	}
+}