chore(RELEASE-1335): add push-disk-images internal task and pipeline (k…

…onflux-ci#738)

This commit moves the push-disk-images pipeline and task from the
app-interface repo to the internal directory of this repo. It
also adds tests and a README with it.

Signed-off-by: Johnny Bieren <[email protected]>

internal/pipelines/push-disk-images/README.md

@@ -0,0 +1,15 @@
+# push-disk-images
+
+Tekton Pipeline to push disk images with Pulp
+
+## Parameters
+
+| Name           | Description                                                       | Optional | Default value                                            |
+|----------------|-------------------------------------------------------------------|----------|----------------------------------------------------------|
+| snapshot_json  | String containing a JSON representation of the snapshot spec      | No       | -                                                        |
+| exodusGwSecret | Env specific secret containing the Exodus Gateway configs         | No       | -                                                        |
+| exodusGwEnv    | Environment to use in the Exodus Gateway. Options are [live, pre] | No       | -                                                        |
+| pulpSecret     | Env specific secret containing the rhsm-pulp credentials          | No       | -                                                        |
+| udcacheSecret  | Env specific secret containing the udcache credentials            | No       | -                                                        |
+| cgwHostname    | The hostname of the content-gateway to publish the metadata to    | Yes      | https://developers.redhat.com/content-gateway/rest/admin |
+| cgwSecret      | Env specific secret containing the content gateway credentials    | No       | -                                                        |

internal/pipelines/push-disk-images/push-disk-images.yaml

@@ -0,0 +1,59 @@
+---
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: push-disk-images
+  labels:
+    app.kubernetes.io/version: "0.2.2"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.12.1"
+    tekton.dev/tags: release
+spec:
+  description: >-
+      Pipeline to push disk images with pulp
+  params:
+    - name: snapshot_json
+      type: string
+      description: String containing a JSON representation of the snapshot spec
+    - name: exodusGwSecret
+      type: string
+      description: Env specific secret containing the Exodus Gateway configs
+    - name: exodusGwEnv
+      type: string
+      description: Environment to use in the Exodus Gateway. Options are [live, pre]
+    - name: pulpSecret
+      type: string
+      description: Env specific secret containing the rhsm-pulp credentials
+    - name: udcacheSecret
+      type: string
+      description: Env specific secret containing the udcache credentials
+    - name: cgwHostname
+      type: string
+      description: The hostname of the content-gateway to publish the metadata to
+      default: https://developers.redhat.com/content-gateway/rest/admin
+    - name: cgwSecret
+      type: string
+      description: Env specific secret containing the content gateway credentials
+  tasks:
+    - name: pulp-push-disk-images
+      timeout: "24h00m0s"
+      taskRef:
+        name: pulp-push-disk-images
+      params:
+        - name: snapshot_json
+          value: $(params.snapshot_json)
+        - name: exodusGwSecret
+          value: $(params.exodusGwSecret)
+        - name: exodusGwEnv
+          value: $(params.exodusGwEnv)
+        - name: pulpSecret
+          value: $(params.pulpSecret)
+        - name: udcacheSecret
+          value: $(params.udcacheSecret)
+        - name: cgwHostname
+          value: $(params.cgwHostname)
+        - name: cgwSecret
+          value: $(params.cgwSecret)
+  results:
+    - name: result
+      value: $(tasks.pulp-push-disk-images.results.result)

internal/resources/pulp-push-disk-images.yaml

@@ -0,0 +1 @@
+../tasks/pulp-push-disk-images/pulp-push-disk-images.yaml

internal/resources/push-disk-images.yaml

@@ -0,0 +1 @@
+../pipelines/push-disk-images/push-disk-images.yaml

internal/tasks/pulp-push-disk-images/README.md

@@ -0,0 +1,16 @@
+# pulp-push-disk-images
+
+Tekton task to push disk images with Pulp
+
+## Parameters
+
+| Name            | Description                                                       | Optional | Default value                                            |
+|-----------------|-------------------------------------------------------------------|----------|----------------------------------------------------------|
+| snapshot_json   | String containing a JSON representation of the snapshot spec      | No       | -                                                        |
+| concurrentLimit | The maximum number of images to be pulled at once                 | Yes      | 3                                                        |
+| exodusGwSecret  | Env specific secret containing the Exodus Gateway configs         | No       | -                                                        |
+| exodusGwEnv     | Environment to use in the Exodus Gateway. Options are [live, pre] | No       | -                                                        |
+| pulpSecret      | Env specific secret containing the rhsm-pulp credentials          | No       | -                                                        |
+| udcacheSecret   | Env specific secret containing the udcache credentials            | No       | -                                                        |
+| cgwHostname     | The hostname of the content-gateway to publish the metadata to    | Yes      | https://developers.redhat.com/content-gateway/rest/admin |
+| cgwSecret       | Env specific secret containing the content gateway credentials    | No       | -                                                        |

internal/tasks/pulp-push-disk-images/pulp-push-disk-images.yaml

@@ -0,0 +1,286 @@
+---
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: pulp-push-disk-images
+  labels:
+    app.kubernetes.io/version: "0.2.2"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.12.1"
+    tekton.dev/tags: release
+spec:
+  description: >-
+    Tekton task to push disk images with pulp
+  params:
+    - name: snapshot_json
+      type: string
+      description: String containing a JSON representation of the snapshot spec
+    - name: concurrentLimit
+      type: string
+      description: The maximum number of images to be pulled at once
+      default: 3
+    - name: exodusGwSecret
+      type: string
+      description: Env specific secret containing the Exodus Gateway configs
+    - name: exodusGwEnv
+      type: string
+      description: Environment to use in the Exodus Gateway. Options are [live, pre]
+    - name: pulpSecret
+      type: string
+      description: Env specific secret containing the rhsm-pulp credentials
+    - name: udcacheSecret
+      type: string
+      description: Env specific secret containing the udcache credentials
+    - name: cgwHostname
+      type: string
+      description: Env specific hostname for content gateway
+    - name: cgwSecret
+      type: string
+      description: Env specific secret containing the content gateway credentials
+  results:
+    - name: result
+      description: Success if the task succeeds, the error otherwise
+  steps:
+    - name: pull-and-push-images
+      image: quay.io/konflux-ci/release-service-utils:6556e8a6b031c1aad4f0472703fd121a6e1cd45d
+      env:
+        - name: EXODUS_CERT
+          valueFrom:
+            secretKeyRef:
+              name: $(params.exodusGwSecret)
+              key: cert
+        - name: EXODUS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: $(params.exodusGwSecret)
+              key: key
+        - name: EXODUS_URL
+          valueFrom:
+            secretKeyRef:
+              name: $(params.exodusGwSecret)
+              key: url
+        - name: PULP_URL
+          valueFrom:
+            secretKeyRef:
+              name: $(params.pulpSecret)
+              key: pulp_url
+        - name: PULP_CERT
+          valueFrom:
+            secretKeyRef:
+              name: $(params.pulpSecret)
+              key: konflux-release-rhsm-pulp.crt
+        - name: PULP_KEY
+          valueFrom:
+            secretKeyRef:
+              name: $(params.pulpSecret)
+              key: konflux-release-rhsm-pulp.key
+        - name: UDC_URL
+          valueFrom:
+            secretKeyRef:
+              name: $(params.udcacheSecret)
+              key: url
+        - name: UDC_CERT
+          valueFrom:
+            secretKeyRef:
+              name: $(params.udcacheSecret)
+              key: cert
+        - name: UDC_KEY
+          valueFrom:
+            secretKeyRef:
+              name: $(params.udcacheSecret)
+              key: key
+        - name: DOCKER_CONFIG_JSON
+          valueFrom:
+            secretKeyRef:
+              name: redhat-workloads-token
+              key: .dockerconfigjson
+        - name: "SNAPSHOT_JSON"
+          value: "$(params.snapshot_json)"
+        - name: CGW_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: $(params.cgwSecret)
+              key: username
+        - name: CGW_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: $(params.cgwSecret)
+              key: token
+      script: |
+        #!/usr/bin/env bash
+        set -ex
+
+        STDERR_FILE=/tmp/stderr.txt
+
+        exitfunc() {
+            local err=$1
+            local line=$2
+            local command="$3"
+            if [ "$err" -eq 0 ] ; then
+                echo -n "Success" > "$(results.result.path)"
+            else
+                echo "$0: ERROR '$command' failed at line $line - exited with status $err" \
+                  > "$(results.result.path)"
+                if [ -f "$STDERR_FILE" ] ; then
+                    tail -n 20 "$STDERR_FILE" >> "$(results.result.path)"
+                fi
+            fi
+            exit 0 # exit the script cleanly as there is no point in proceeding past an error or exit call
+        }
+        # due to set -e, this catches all EXIT and ERR calls and the task should never fail with nonzero exit code
+        trap 'exitfunc $? $LINENO "$BASH_COMMAND"' EXIT
+
+        # Setup required variables
+        export EXODUS_GW_CERT=/tmp/exodus.crt
+        export EXODUS_GW_KEY=/tmp/exodus.key
+        export PULP_CERT_FILE=/tmp/pulp.crt
+        export PULP_KEY_FILE=/tmp/pulp.key
+        export UDCACHE_CERT=/tmp/udc.crt
+        export UDCACHE_KEY=/tmp/udc.key
+        EXODUS_GW_ENV=$(params.exodusGwEnv)
+        export EXODUS_GW_ENV
+        export EXODUS_GW_URL="$EXODUS_URL"
+        export EXODUS_PULP_HOOK_ENABLED=True
+        export EXODUS_GW_TIMEOUT=7200
+        mkdir -p ~/.docker
+
+        set +x
+        echo "$EXODUS_CERT" > "$EXODUS_GW_CERT"
+        echo "$EXODUS_KEY" > "$EXODUS_GW_KEY"
+        echo "$PULP_CERT" > "$PULP_CERT_FILE"
+        echo "$PULP_KEY" > "$PULP_KEY_FILE"
+        echo "$UDC_CERT" > "$UDCACHE_CERT"
+        echo "$UDC_KEY" > "$UDCACHE_KEY"
+        # Quotes are added to the secret so it applies in k8s nicely. But now we have to remove them
+        echo "$DOCKER_CONFIG_JSON" | sed -r 's/(^|\})[^{}]+(\{|$)/\1\2/g' > ~/.docker/config.json
+        set -x
+
+        DISK_IMAGE_DIR="$(mktemp -d)"
+        export DISK_IMAGE_DIR
+
+        process_component() { # Expected argument is [component json]
+            COMPONENT=$1
+            PULLSPEC=$(jq -er '.containerImage' <<< "${COMPONENT}")
+            DESTINATION="${DISK_IMAGE_DIR}/$(jq -er '.staged.destination' <<< "${COMPONENT}")/FILES" \
+              || (echo "Missing staged.destination value for component. This should be an existing pulp repo. \
+                  Failing" && exit 1)
+            mkdir -p "${DESTINATION}"
+            DOWNLOAD_DIR=$(mktemp -d)
+            cd "$DOWNLOAD_DIR"
+            # oras has very limited support for selecting the right auth entry,
+            # so create a custom auth file with just one entry
+            AUTH_FILE=$(mktemp)
+            select-oci-auth "${PULLSPEC}" > "$AUTH_FILE"
+            oras pull --registry-config "$AUTH_FILE" "$PULLSPEC"
+            NUM_MAPPED_FILES=$(jq '.staged.files | length' <<< "${COMPONENT}")
+            for ((i = 0; i < NUM_MAPPED_FILES; i++)) ; do
+                FILE=$(jq -c --arg i "$i" '.staged.files[$i|tonumber]' <<< "$COMPONENT")
+                SOURCE=$(jq -er '.source' <<< "$FILE")
+                FILENAME=$(jq -er '.filename' <<< "$FILE")
+                # The .qcow2 images are not zipped
+                if [ -f "${SOURCE}.gz" ] ; then
+                    gzip -d "${SOURCE}.gz"
+                fi
+                DESTINATION_FILE="${DESTINATION}/${FILENAME}"
+                # Albeit a rare one, this is a race condition since this is run in parallel.
+                # The race condition is if two files have the same $DESTINATION_FILE and both
+                # if checks are run before either mv is run a few lines below.
+                if [ -f "${DESTINATION_FILE}" ] ; then
+                    echo -n "Multiple files use the same destination value: $DESTINATION" >&2
+                    echo " and filename value: $FILENAME. Failing..." >&2
+                    exit 1
+                fi
+                mv "$SOURCE" "${DESTINATION_FILE}" || echo "didn't find mapped file: ${SOURCE}"
+            done
+
+        }
+
+        process_component_for_developer_portal() { # Expected argument are [component json], [content_directory]
+          COMPONENT=$1
+
+          productName="$(jq -er '.contentGateway.productName' <<< "${COMPONENT}")" \
+            || (echo "Missing contentGateway.productName value for component. This should be an existing product \
+                  in the Developer Portal. Failing" && exit 1)
+
+          productCode="$(jq -er '.contentGateway.productCode' <<< "${COMPONENT}")" \
+            || (echo "Missing contentGateway.productCode value for component. This should be an existing product \
+                  in the Developer Portal. Failing" && exit 1)
+
+          productVersionName="$(jq -er '.contentGateway.productVersionName' <<< "${COMPONENT}")" \
+            || (echo "Missing contentGateway.productVersionName value for component. This should be an existing \
+                  product in the Developer Portal. Failing" && exit 1)
+
+          filePrefix="$(jq -er '.contentGateway.filePrefix' <<< "${COMPONENT}")" \
+            || (echo "Missing contentGateway.filePrefix value for component. This should be the prefix for files to \
+                  upload to the Developer Portal. Failing" && exit 1)
+
+          developer_portal_wrapper --debug --product-name "${productName}" \
+            --product-code "${productCode}" \
+            --product-version-name "${productVersionName}" \
+            --cgw-hostname "$(params.cgwHostname)" \
+            --content-directory "$2" \
+            --file-prefix "${filePrefix}"
+
+        }
+
+        RUNNING_JOBS="\j" # Bash parameter for number of jobs currently running
+        NUM_COMPONENTS=$(jq '.components | length' <<< "$SNAPSHOT_JSON")
+
+        # use the 1st component's version
+        VERSION=$(jq -cr '.components[0].staged.version // ""' <<< "$SNAPSHOT_JSON")
+        if [ "${VERSION}" == "" ] ; then
+          echo "Error: version not specified in .components[0].staged.version. Needed to publish to customer portal"
+          exit 1
+        fi
+
+        # Process each component in parallel
+        for ((i = 0; i < NUM_COMPONENTS; i++)) ; do
+            COMPONENT=$(jq -c --arg i "$i" '.components[$i|tonumber]' <<< "$SNAPSHOT_JSON")
+            # Limit batch size to concurrent limit
+            while (( ${RUNNING_JOBS@P} >= $(params.concurrentLimit) )); do
+                wait -n
+            done
+            process_component "$COMPONENT" 2> "$STDERR_FILE" &
+        done
+
+        # Wait for remaining processes to finish
+        while (( ${RUNNING_JOBS@P} > 0 )); do
+            wait -n
+        done
+
+        # Change to the subdir with the images
+        cd "${DISK_IMAGE_DIR}"
+
+        STAGED_JSON='{"header":{"version": "0.2"},"payload":{"files":[]}}'
+
+        # Add the files to the payload
+        # shell check wants us to find ./* but that adds `./` to the paths which breaks the script
+        # shellcheck disable=SC2035
+        while IFS= read -r -d '' file ; do
+            STAGED_JSON=$(jq --arg filename "$(basename "$file")" --arg path "$file" \
+              --arg version "$VERSION" \
+              '.payload.files[.payload.files | length] = 
+              {"filename": $filename, "relative_path": $path, "version": $version}' <<< "$STAGED_JSON")
+        done < <(find * -type f -print0)
+
+        echo "$STAGED_JSON" | yq -P -I 4 > staged.yaml
+
+        pulp_push_wrapper --debug --source "${DISK_IMAGE_DIR}" --pulp-url "$PULP_URL" \
+          --pulp-cert $PULP_CERT_FILE --pulp-key $PULP_KEY_FILE --udcache-url "$UDC_URL" \
+          2> "$STDERR_FILE"
+
+        relative_paths=$(echo "$STAGED_JSON" | jq -erc .payload.files[].relative_path)
+        component_destinations=()
+        for path in $relative_paths:
+        do
+          parent_dir=$(dirname "$path")
+          component_destinations+=("${DISK_IMAGE_DIR}/$parent_dir")
+        done
+
+        ## Process Files for Developer Portal / CGW
+        ##
+        NUM_COMPONENTS=$(jq '.components | length' <<< "$SNAPSHOT_JSON")
+        for ((i = 0; i < NUM_COMPONENTS; i++)) ; do
+            COMPONENT=$(jq -c --arg i "$i" '.components[$i|tonumber]' <<< "$SNAPSHOT_JSON")
+            process_component_for_developer_portal "$COMPONENT" "${component_destinations[$i]}"  2> "$STDERR_FILE"
+        done