Use the Makefile to build and push in GH actions (#313)

Signed-off-by: Roy Golan <[email protected]>
rhdhorchestrator · Jul 22, 2024 · 5f784c3 · masayag · Jul 28, 2024 · 5f784c3
1 parent e409fc9
commit 5f784c3
Show file tree

Hide file tree

Showing 15 changed files with 179 additions and 172 deletions.
diff --git a/.github/workflows/jira-listener.yml b/.github/workflows/jira-listener.yml
@@ -17,4 +17,4 @@ jobs:
     with:
       workflow_id: escalation
       application_id: jira-listener
-    secrets: inherit
+    secrets: inherit
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -6,6 +6,13 @@ on:
       workflow_id:
         required: true
         type: string
+      # application is those auxilary application we deploy with a workflow to
+      # complete some part of it with specialized code, like for example the
+      # jira-listener for the escalation flow.
+      application_id:
+        required: false
+        type: string
+        default: 'UNDEFINED'
       it_mode:
         type: boolean
         default: false
@@ -28,23 +35,18 @@ jobs:
         username: ${{ secrets.REGISTRY_REDHAT_IO_USER }}
         password: ${{ secrets.REGISTRY_REDHAT_IO_PASSWORD }}
 
-    - name: Buildah Action
+    - name: Build Image
       id: build-image
-      uses: redhat-actions/buildah-build@v2
       if: ${{ ! inputs.it_mode }}
-      with:
-        image: serverless-workflow-${{ inputs.workflow_id }}
-        tags: latest ${{ github.sha }}
-        extra-args: --ulimit nofile=4096:4096
-        containerfiles: |
-          pipeline/workflow-builder.Dockerfile
-        build-args: |
-          WF_RESOURCES=${{ inputs.workflow_id }}
-          FLOW_NAME=${{ inputs.workflow_id }}
-          FLOW_SUMMARY=${{ inputs.workflow_id }}
-          FLOW_DESCRIPTION=${{ inputs.workflow_id }}
-          MAVEN_ARGS_APPEND=-Dkogito.persistence.type=jdbc -Dquarkus.datasource.db-kind=postgresql -Dkogito.persistence.proto.marshaller=false
-          QUARKUS_EXTENSIONS=org.kie.kogito:kogito-addons-quarkus-jobs-knative-eventing:9.99.1.redhat-00003,org.kie.kogito:kogito-addons-quarkus-persistence-jdbc:9.99.1.redhat-00003,org.kie.kogito:kogito-addons-persistence-jdbc:9.99.1.redhat-00003,io.quarkus:quarkus-jdbc-postgresql:3.2.9.Final,io.quarkus:quarkus-agroal:3.2.9.Final,org.kie:kie-addons-quarkus-monitoring-prometheus:999-SNAPSHOT,org.kie:kie-addons-quarkus-monitoring-sonataflow:999-SNAPSHOT
+      run: |
+        make WORKFLOW_ID=${{ inputs.workflow_id }} \
+          APPLICATION_ID=${{ inputs.application_id }} \
+          IMAGE_TAG=${{ github.sha }} \
+          GIT_TOKEN=${{ env.GH_TOKEN }} \
+          REGISTRY_REPO=${{ env.REGISTRY_REPO }} \
+          REGISTRY_USERNAME=${{ secrets.NEW_QUAY_USERNAME }} \
+          REGISTRY_PASSWORD=${{ secrets.NEW_QUAY_PASSWORD }} \
+          build-image
 
     - name: Buildah Action for testing
       id: build-dev-image
@@ -61,8 +63,6 @@ jobs:
           FLOW_NAME=${{ inputs.workflow_id }}
           FLOW_SUMMARY=${{ inputs.workflow_id }}
           FLOW_DESCRIPTION=${{ inputs.workflow_id }}
-          MAVEN_ARGS_APPEND=
-          QUARKUS_EXTENSIONS=
 
     - name: Buildah push to OCI Arcive
       if: ${{ inputs.it_mode }}
@@ -79,70 +79,38 @@ jobs:
     - name: Push To quay.io
       id: push-to-quay
       if: ${{ ! inputs.it_mode }}
-      uses: redhat-actions/push-to-registry@v2
-      with:
-        image: ${{ steps.build-image.outputs.image }}
-        tags: ${{ steps.build-image.outputs.tags }}
-        registry: quay.io/orchestrator
-        username: ${{ secrets.NEW_QUAY_USERNAME }}
-        password: ${{ secrets.NEW_QUAY_PASSWORD }}
-
-    - name: Print image url
-      run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"
+      run: |
+        make WORKFLOW_ID=${{ inputs.workflow_id }} \
+          APPLICATION_ID=${{ inputs.application_id }} \
+          IMAGE_TAG=${{ github.sha }} \
+          GIT_TOKEN=${{ env.GH_TOKEN }} \
+          REGISTRY_REPO=${{ env.REGISTRY_REPO }} \
+          REGISTRY_USERNAME=${{ secrets.NEW_QUAY_USERNAME }} \
+          REGISTRY_PASSWORD=${{ secrets.NEW_QUAY_PASSWORD }} \
+          push-image
 
   generate-manifests:
     needs: build
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: write
     steps:
     - uses: actions/checkout@v4
-
-    - name: Add auto DB migration in props
-      run: echo -e "\nquarkus.flyway.migrate-at-start=true" >> ${{ inputs.workflow_id }}/application.properties
-
-    - name: Setup kn-workflow
-      run: |
-          KN_CLI_URL="https://mirror.openshift.com/pub/openshift-v4/clients/serverless/1.11.2/kn-linux-amd64.tar.gz"
-          curl -L "$KN_CLI_URL" | tar -xz && chmod +x kn-linux-amd64 && mv kn-linux-amd64 kn
-
-    - name: Use kn-workflow to generate manifests
-      run: cd ${{ inputs.workflow_id }} && ../kn workflow gen-manifest --namespace ""
-
-    - name: Set prod profile
-      run: yq --inplace eval '.metadata.annotations["sonataflow.org/profile"] = "prod"' ${{ inputs.workflow_id }}/manifests/01-sonataflow*.yaml
-
-    - name: Set container image ref in SonataFlow resource
-      run: yq --inplace '.spec.podTemplate.container.image="quay.io/orchestrator/serverless-workflow-${{ inputs.workflow_id }}:${{ github.sha }}"' ${{ inputs.workflow_id }}/manifests/01-sonataflow*.yaml
-
-    - name: Set persistence spec in SonataFlow resource
+    - name: Generate Manifests
+      id: generate-manifests
       run: |
-        yq --inplace '.spec |= (
-          . + {
-            "persistence": {
-              "postgresql": {
-                "secretRef": {
-                  "name": "sonataflow-psql-postgresql",
-                  "userKey": "postgres-username",
-                  "passwordKey": "postgres-password"
-                },
-                "serviceRef": {
-                  "name": "sonataflow-psql-postgresql",
-                  "port": 5432,
-                  "databaseName": "sonataflow",
-                  "databaseSchema": "${{ inputs.workflow_id }}"
-                }
-              }
-            }
-          }
-        )' ${{ inputs.workflow_id }}/manifests/01-sonataflow*.yaml
+        WORKDIR=${{ runner.temp }}/serverless-workflows
+        make WORKFLOW_ID=${{ inputs.workflow_id }} \
+          WORKDIR=${WORKDIR} \
+          IMAGE_TAG=${{ github.sha }} \
+          APPLICATION_ID=${{ inputs.application_id }} \
+          PR_OR_COMMIT_URL=${PR_OR_COMMIT_URL} \
+          gen-manifests
+        echo manifests_dir=${{ runner.temp }}/serverless-workflows/${{ inputs.workflow_id }}/manifests >> "$GITHUB_OUTPUT"
 
     - name: Archive manifests
       uses: actions/upload-artifact@v4
       with:
         name: serverless-workflow-${{ inputs.workflow_id }}-manifests
-        path: ${{ inputs.workflow_id }}/manifests
+        path: ${{ steps.generate-manifests.outputs.manifests_dir }}
 
     - uses: actions/github-script@v7
       id: get_pr_data

diff --git a/.github/workflows/move2kube-e2e.yaml b/.github/workflows/move2kube-e2e.yaml
@@ -79,10 +79,9 @@ jobs:
 
       - name: Deploy sonataflow-operator
         run: |
-          helm repo add janus-idp-workflows https://rgolangh.github.io/janus-idp-workflows-helm/
-          helm install janus-idp-workflows janus-idp-workflows/janus-idp-workflows \
-            --set backstage.upstream.backstage.image.tag=1.1 \
-            -f https://raw.githubusercontent.com/rgolangh/janus-idp-workflows-helm/main/charts/kubernetes/orchestrator/values-k8s.yaml
+          helm repo add orchestrator https://parodos-dev.github.io/orchestrator-helm-chart
+          helm install orchestrator orchestrator/orchestrator-k8s
+
           echo "sleep bit long till the PV for data index and kaniko cache is ready. its a bit slow. TODO fixit"
           kubectl get pv
           sleep 3m
@@ -91,6 +90,11 @@ jobs:
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=sonataflow-platform" --timeout=180s
 
+      - name: Deploy fake notifications service
+        run: |
+          source e2e/helper-functions.sh
+          create-fake-notifications-service
+
       - name: Deploy Move2kube serverless workflow
         run: |
           kubectl patch configmap/config-features \
@@ -107,6 +111,11 @@ jobs:
             sed -i '/quarkus\.flyway\.migrate-at-start=true/d' "$file"
           done
 
+          # Set notification external access token
+          echo " Set notification external access token"
+          yq --inplace '.data."NOTIFICATIONS_BEARER_TOKEN" = "ZTJldG9rZW4="' manifests/01-secret_m2k.yaml # notsecret
+          cat manifests/01-secret_m2k.yaml
+
           # deploy the manifests created by the  ${{ steps.build-image.outputs.image }}"                                                                    
           kubectl apply -f manifests
           sleep 5
@@ -118,7 +127,7 @@ jobs:
           
           kubectl patch configmap/m2k-props \
             --type merge \
-            -p '{"data": {"application.properties" :"move2kube_url=http://move2kube-instance-svc.default.svc.cluster.local:8080\nquarkus.rest-client.move2kube_yaml.url=http://move2kube-instance-svc.default.svc.cluster.local:8080\nquarkus.rest-client.notifications.url=http://janus-idp-workflows-backstage.default.svc.cluster.local:7007/api/notifications/"}}'
+            -p '{"data": {"application.properties" :"move2kube_url=http://move2kube-instance-svc.default.svc.cluster.local:8080\nquarkus.rest-client.move2kube_yaml.url=http://move2kube-instance-svc.default.svc.cluster.local:8080\nquarkus.rest-client.notifications.url=http://fake-notifications-service.default.svc.cluster.local:8080/api/notifications/"}}'
           kubectl delete pod -l "app=m2k"
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=m2k" --timeout=1m

diff --git a/.github/workflows/mta-e2e.yaml b/.github/workflows/mta-e2e.yaml
@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+      - uses: actions/checkout@v4
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
         with:
@@ -68,12 +69,10 @@ jobs:
           kubectl wait --for=condition=Ready=true pods -l "app.kubernetes.io/name=tackle-ui" -n my-konveyor-operator --timeout=120s
           # now MTA workflow can execute agains tackle-ui.my-konveyor-operator.svc:8080
 
-      - name: Deploy Janus-idp-workflow-helm (janus + sonataflow-opertor)
+      - name: Deploy Orchestrator-k8s helm chart
         run: |
-          helm repo add janus-idp-workflows https://rgolangh.github.io/janus-idp-workflows-helm/
-          helm install janus-idp-workflows janus-idp-workflows/janus-idp-workflows \
-            --set backstage.upstream.backstage.image.tag=1.1 \
-            -f https://raw.githubusercontent.com/rgolangh/janus-idp-workflows-helm/main/charts/kubernetes/orchestrator/values-k8s.yaml
+          helm repo add orchestrator https://parodos-dev.github.io/orchestrator-helm-chart
+          helm install orchestrator orchestrator/orchestrator-k8s
 
           echo "sleep bit long till the PV for data index and kaniko cache is ready. its a bit slow. TODO fixit"
           kubectl get pv
@@ -82,6 +81,14 @@ jobs:
           kubectl wait --for=condition=Ready=true pods -l "app.kubernetes.io/name=backstage" --timeout=120s
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=sonataflow-platform" --timeout=120s
+          #give the data-index time to register and start.
+          sleep 1m
+          kubectl get pods
+          kubecel get svc
+      - name: Deploy fake notifications service
+        run: |
+          source e2e/helper-functions.sh
+          create-fake-notifications-service
 
       - name: Download sonataflow artifacts generated manifests
         uses: actions/download-artifact@v4
@@ -101,12 +108,8 @@ jobs:
 
       - name: Deploy MTA serverless workflow
         run: |
-          ###### workaround till https://issues.redhat.com/browse/FLPATH-892 is solved
-          # yq --inplace '.spec.podTemplate.container |= ( . +  {"imagePullPolicy": "IfNotPresent"} )' manifests/01-sonataflow_mtaanalysis.yaml
-          ###### end workfaround
-
           # Set the endpoint to the tackle-ui service
-          yq --inplace '.spec.podTemplate.container.env |= ( . + [{"name": "QUARKUS_REST_CLIENT_MTA_JSON_URL", "value": "http://tackle-ui.my-konveyor-operator.svc:8080/hub"}, {"name": "BACKSTAGE_NOTIFICATIONS_URL", "value": "http://janus-idp-workflows-backstage.default.svc.cluster.local:7007/api/notifications/"}] )' manifests/01-sonataflow_mtaanalysis.yaml
+          yq --inplace '.spec.podTemplate.container.env |= ( . + [{"name": "QUARKUS_REST_CLIENT_MTA_JSON_URL", "value": "http://tackle-ui.my-konveyor-operator.svc:8080/hub"}, {"name": "BACKSTAGE_NOTIFICATIONS_URL", "value": "http://fake-notifications-service.default.svc.cluster.local:8080/api/notifications/"}] )' manifests/01-sonataflow_mtaanalysis.yaml
           
           # Disable persistence for e2e tests
           yq e '.spec.persistence = {}' -i manifests/01-sonataflow_mtaanalysis.yaml
@@ -120,6 +123,11 @@ jobs:
           cat manifests/01-sonataflow_mtaanalysis.yaml
           echo "---"
 
+          # Set notification external access token
+          echo " Set notification external access token"
+          yq --inplace '.data."NOTIFICATIONS_BEARER_TOKEN" = "ZTJldG9rZW4="' manifests/01-secret_mtaanalysis.yaml # notsecret
+          cat manifests/01-secret_mtaanalysis.yaml
+
           # deploy the manifests created by the ${{ steps.build-image.outputs.image }} image
           kubectl apply -f manifests/
           sleep 5
@@ -129,7 +137,6 @@ jobs:
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=mtaanalysis" --timeout=5m
 
-      - uses: actions/checkout@v4
       - name: Run e2e script
         run: |
           e2e/mta.sh
@@ -144,4 +151,4 @@ jobs:
         if: always()
         with:
           name: kind-logs
-          path: ./kind_logs/
+          path: ./kind_logs/
diff --git a/.github/workflows/mta-v6.x-e2e.yaml b/.github/workflows/mta-v6.x-e2e.yaml
@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+      - uses: actions/checkout@v4
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
         with:
@@ -70,10 +71,8 @@ jobs:
 
       - name: Deploy Janus-idp-workflow-helm (janus + sonataflow-opertor)
         run: |
-          helm repo add janus-idp-workflows https://rgolangh.github.io/janus-idp-workflows-helm/
-          helm install janus-idp-workflows janus-idp-workflows/janus-idp-workflows \
-            --set backstage.upstream.backstage.image.tag=1.1 \
-            -f https://raw.githubusercontent.com/rgolangh/janus-idp-workflows-helm/main/charts/kubernetes/orchestrator/values-k8s.yaml
+          helm repo add orchestrator https://parodos-dev.github.io/orchestrator-helm-chart
+          helm install orchestrator orchestrator/orchestrator-k8s
 
           echo "sleep bit long till the PV for data index and kaniko cache is ready. its a bit slow. TODO fixit"
           kubectl get pv
@@ -83,6 +82,11 @@ jobs:
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=sonataflow-platform" --timeout=600s
 
+      - name: Deploy fake notifications service
+        run: |
+          source e2e/helper-functions.sh
+          create-fake-notifications-service
+
       - name: Download sonataflow artifacts generated manifests
         uses: actions/download-artifact@v4
         with:
@@ -101,12 +105,8 @@ jobs:
 
       - name: Deploy MTA serverless workflow
         run: |
-          ###### workaround till https://issues.redhat.com/browse/FLPATH-892 is solved
-          # yq --inplace '.spec.podTemplate.container |= ( . +  {"imagePullPolicy": "IfNotPresent"} )' manifests/01-sonataflow_mta-analysis-v6.yaml
-          ###### end workfaround
-
           # Set the endpoint to the tackle-ui service
-          yq --inplace '.spec.podTemplate.container.env |= ( . + [{"name": "QUARKUS_REST_CLIENT_MTA_JSON_URL", "value": "http://tackle-ui.my-konveyor-operator.svc:8080"}, {"name": "BACKSTAGE_NOTIFICATIONS_URL", "value": "http://janus-idp-workflows-backstage.default.svc.cluster.local:7007/api/notifications/"}] )' manifests/01-sonataflow_mta-analysis-v6.yaml
+          yq --inplace '.spec.podTemplate.container.env |= ( . + [{"name": "QUARKUS_REST_CLIENT_MTA_JSON_URL", "value": "http://tackle-ui.my-konveyor-operator.svc:8080"}, {"name": "BACKSTAGE_NOTIFICATIONS_URL", "value": "http://fake-notifications-service.default.svc.cluster.local:8080/api/notifications/"}] )' manifests/01-sonataflow_mta-analysis-v6.yaml
           
           # Disable persistence for e2e tests
           yq e '.spec.persistence = {}' -i manifests/01-sonataflow_mta-analysis-v6.yaml
@@ -120,6 +120,11 @@ jobs:
           cat manifests/01-sonataflow_mta-analysis-v6.yaml
           echo "---"
 
+          # Set notification external access token
+          echo " Set notification external access token"
+          yq --inplace '.data."NOTIFICATIONS_BEARER_TOKEN" = "ZTJldG9rZW4="' manifests/01-secret_mta-analysis-v6.yaml # notsecret
+          cat manifests/01-secret_mta-analysis-v6.yaml
+
           # deploy the manifests created by the ${{ steps.build-image.outputs.image }} image
           kubectl apply -f manifests/
           sleep 5
@@ -129,7 +134,6 @@ jobs:
           kubectl get pods -o wide
           kubectl wait --for=condition=Ready=true pods -l "app=mta-analysis-v6" --timeout=10m
 
-      - uses: actions/checkout@v4
       - name: Run e2e script
         run: |
           e2e/mta-v6.x.sh

diff --git a/.github/workflows/mta-v7.x-e2e.yaml b/.github/workflows/mta-v7.x-e2e.yaml
@@ -114,10 +114,6 @@ jobs:
 
       - name: Deploy MTA serverless workflow
         run: |
-          ###### workaround till https://issues.redhat.com/browse/FLPATH-892 is solved
-          # yq --inplace '.spec.podTemplate.container |= ( . +  {"imagePullPolicy": "IfNotPresent"} )' manifests/01-sonataflow_mta-analysis-v7.yaml
-          ###### end workfaround
-
           # Set the endpoint to the tackle-ui service
           yq --inplace '.spec.podTemplate.container.env |= ( . + [{"name": "QUARKUS_REST_CLIENT_MTA_JSON_URL", "value": "http://tackle-ui.my-konveyor-operator.svc:8080"}, {"name": "BACKSTAGE_NOTIFICATIONS_URL", "value": "http://janus-idp-workflows-backstage.default.svc.cluster.local:7007/api/notifications/"}] )' manifests/01-sonataflow_mta-analysis-v7.yaml