From 10f41a9e750fe033d867df90ab126e9348da9c2d Mon Sep 17 00:00:00 2001 From: Luiz Carvalho Date: Thu, 12 Sep 2024 17:06:26 -0400 Subject: [PATCH 1/4] Include Image Index digest in PROCESSED_IMAGES In order to fix KFLUXBUGS-1616, we need to ensure the digest of the Image Index is also included in the list of PROCESSED_IMAGES. This will allow EC to verify the Image Index directly as well as the Image Manifests. Signed-off-by: Luiz Carvalho --- task/clair-scan/0.1/clair-scan.yaml | 6 +++++- task/clamav-scan/0.1/clamav-scan.yaml | 5 +++++ task/deprecated-image-check/0.4/deprecated-image-check.yaml | 6 ++++++ task/sbom-json-check/0.1/sbom-json-check.yaml | 5 +++++ task/sbom-json-check/0.2/sbom-json-check.yaml | 5 +++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/task/clair-scan/0.1/clair-scan.yaml b/task/clair-scan/0.1/clair-scan.yaml index 3701dffe71..e87d976e34 100644 --- a/task/clair-scan/0.1/clair-scan.yaml +++ b/task/clair-scan/0.1/clair-scan.yaml @@ -113,9 +113,13 @@ spec: fi done + # If the image is an Image Index, also add the Image Index digest to the list. + if [[ "${digests_processed[*]}" != *"$IMAGE_DIGEST"* ]]; then + digests_processed+=("\"$IMAGE_DIGEST\"") + fi + digests_processed_string=$(IFS=,; echo "${digests_processed[*]}") - # add the image_index to the processed digests list and store the result in a file images_processed=$(echo "${images_processed_template/\[%s]/[$digests_processed_string]}") echo "$images_processed" > /tekton/home/images-processed.json - name: conftest-vulnerabilities diff --git a/task/clamav-scan/0.1/clamav-scan.yaml b/task/clamav-scan/0.1/clamav-scan.yaml index 3ccf543be2..6a344b22b2 100644 --- a/task/clamav-scan/0.1/clamav-scan.yaml +++ b/task/clamav-scan/0.1/clamav-scan.yaml @@ -145,6 +145,11 @@ spec: "note" : (if .result == "" or ($item.result == "SKIPPED" and .result == "SUCCESS") or ($item.result == "WARNING" and (.result == "SUCCESS" or .result == "SKIPPED")) or ($item.result == "FAILURE" and .result != "ERROR") or $item.result == "ERROR" then $item.note else .note end) })' /work/logs/clamscan-ec-test-*.json | tee $(results.TEST_OUTPUT.path) + # If the image is an Image Index, also add the Image Index digest to the list. + if [[ "${digests_processed[*]}" != *"$IMAGE_DIGEST"* ]]; then + digests_processed+=("\"$IMAGE_DIGEST\"") + fi + digests_processed_string=$(IFS=,; echo "${digests_processed[*]}") echo "${images_processed_template/\[%s]/[$digests_processed_string]}" | tee $(results.IMAGES_PROCESSED.path) volumeMounts: diff --git a/task/deprecated-image-check/0.4/deprecated-image-check.yaml b/task/deprecated-image-check/0.4/deprecated-image-check.yaml index 2e0f5eec72..222753f873 100644 --- a/task/deprecated-image-check/0.4/deprecated-image-check.yaml +++ b/task/deprecated-image-check/0.4/deprecated-image-check.yaml @@ -99,6 +99,12 @@ spec: digests_processed+=("\"$arch_sha\"") done < <(echo "$image_manifests" | jq -r 'to_entries[] | "\(.key) \(.value)"') fi + + # If the image is an Image Index, also add the Image Index digest to the list. + if [[ "${digests_processed[*]}" != *"$IMAGE_DIGEST"* ]]; then + digests_processed+=("\"$IMAGE_DIGEST\"") + fi + digests_processed_string=$(IFS=,; echo "${digests_processed[*]}") if [ -n "${BASE_IMAGES_DIGESTS}" ]; diff --git a/task/sbom-json-check/0.1/sbom-json-check.yaml b/task/sbom-json-check/0.1/sbom-json-check.yaml index 94ac46b851..e17ca78c44 100644 --- a/task/sbom-json-check/0.1/sbom-json-check.yaml +++ b/task/sbom-json-check/0.1/sbom-json-check.yaml @@ -84,6 +84,11 @@ spec: done < <(echo "$image_manifests" | jq -r 'to_entries[] | "\(.key) \(.value)"') fi + # If the image is an Image Index, also add the Image Index digest to the list. + if [[ "${digests_processed[*]}" != *"$IMAGE_DIGEST"* ]]; then + digests_processed+=("\"$IMAGE_DIGEST\"") + fi + # arrays to keep count of successful and failed checks successes=() failures=() diff --git a/task/sbom-json-check/0.2/sbom-json-check.yaml b/task/sbom-json-check/0.2/sbom-json-check.yaml index 94ac46b851..e17ca78c44 100644 --- a/task/sbom-json-check/0.2/sbom-json-check.yaml +++ b/task/sbom-json-check/0.2/sbom-json-check.yaml @@ -84,6 +84,11 @@ spec: done < <(echo "$image_manifests" | jq -r 'to_entries[] | "\(.key) \(.value)"') fi + # If the image is an Image Index, also add the Image Index digest to the list. + if [[ "${digests_processed[*]}" != *"$IMAGE_DIGEST"* ]]; then + digests_processed+=("\"$IMAGE_DIGEST\"") + fi + # arrays to keep count of successful and failed checks successes=() failures=() From 2df72f5ac1c7348a7365f08d9349833042ba7d03 Mon Sep 17 00:00:00 2001 From: Tomas Nevrlka Date: Tue, 3 Sep 2024 13:48:03 +0200 Subject: [PATCH 2/4] replace cosign image with appstudio-utils redhat-appstudio/cosign image is getting deprecated due to migration to konflux-ci, replace with konflux-ci/appstudio-utils image --- renovate.json | 1 - task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml | 12 +++--------- task/oci-copy/0.1/oci-copy.yaml | 12 +++--------- task/rpm-ostree/0.1/rpm-ostree.yaml | 17 ++++------------- task/rpm-ostree/0.2/rpm-ostree.yaml | 17 ++++------------- task/s2i-java/0.1/s2i-java.yaml | 14 +++----------- task/s2i-nodejs/0.1/s2i-nodejs.yaml | 14 +++----------- 7 files changed, 20 insertions(+), 67 deletions(-) diff --git a/renovate.json b/renovate.json index 4426b2f42c..c1769a4a0d 100644 --- a/renovate.json +++ b/renovate.json @@ -25,7 +25,6 @@ "quay.io/redhat-appstudio/buildah", "quay.io/redhat-appstudio/hacbs-jvm-build-request-processor", "quay.io/redhat-appstudio/build-definitions-source-image-build-utils", - "quay.io/redhat-appstudio/cosign", "quay.io/redhat-appstudio/cachi2", "quay.io/redhat-appstudio/sbom-utility-scripts-image", "registry.access.redhat.com/rh-syft-tech-preview/syft-rhel9" diff --git a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml index 8038cd40a6..e0f995b1ab 100644 --- a/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml +++ b/task/oci-copy-oci-ta/0.1/oci-copy-oci-ta.yaml @@ -315,16 +315,10 @@ spec: yq -oj -i '.components += [ {"purl": "'$purl'", "type": "file", "name": "'$OCI_FILENAME'", "hashes": [{"alg": "SHA-256", "content": "'$OCI_ARTIFACT_DIGEST'"}], "externalReferences": [{"type": "distribution", "url": "'$OCI_SOURCE'"}]} ]' sbom-cyclonedx.json done - name: upload-sbom - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 workingDir: /var/workdir + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" - name: report-sbom-url image: quay.io/konflux-ci/yq:latest@sha256:f758d9a25bc88cc114bfb6137fd4d649db427de5a4217e818b8466ad5bf9255c workingDir: /var/workdir diff --git a/task/oci-copy/0.1/oci-copy.yaml b/task/oci-copy/0.1/oci-copy.yaml index 1b2bdc9b4c..dfe06b5e88 100644 --- a/task/oci-copy/0.1/oci-copy.yaml +++ b/task/oci-copy/0.1/oci-copy.yaml @@ -295,16 +295,10 @@ spec: done workingDir: $(workspaces.source.path) - name: upload-sbom - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 workingDir: $(workspaces.source.path) + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" - name: report-sbom-url image: quay.io/konflux-ci/yq:latest@sha256:f758d9a25bc88cc114bfb6137fd4d649db427de5a4217e818b8466ad5bf9255c script: | diff --git a/task/rpm-ostree/0.1/rpm-ostree.yaml b/task/rpm-ostree/0.1/rpm-ostree.yaml index 34c65c2635..1a17b1ac41 100644 --- a/task/rpm-ostree/0.1/rpm-ostree.yaml +++ b/task/rpm-ostree/0.1/rpm-ostree.yaml @@ -281,20 +281,11 @@ spec: - mountPath: /var/lib/containers name: varlibcontainers workingDir: $(workspaces.source.path) - - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting - # the cluster will set imagePullPolicy to IfNotPresent - name: upload-sbom - computeResources: {} + - name: upload-sbom + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 workingDir: $(workspaces.source.path) + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" volumes: - emptyDir: {} name: varlibcontainers diff --git a/task/rpm-ostree/0.2/rpm-ostree.yaml b/task/rpm-ostree/0.2/rpm-ostree.yaml index d82ddd6ad7..3414482a53 100644 --- a/task/rpm-ostree/0.2/rpm-ostree.yaml +++ b/task/rpm-ostree/0.2/rpm-ostree.yaml @@ -276,20 +276,11 @@ spec: - mountPath: /var/lib/containers name: varlibcontainers workingDir: $(workspaces.source.path) - - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting - # the cluster will set imagePullPolicy to IfNotPresent - name: upload-sbom - computeResources: {} + - name: upload-sbom + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 workingDir: $(workspaces.source.path) + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" volumes: - emptyDir: {} name: varlibcontainers diff --git a/task/s2i-java/0.1/s2i-java.yaml b/task/s2i-java/0.1/s2i-java.yaml index 7f1d1d3cd0..707cc86472 100644 --- a/task/s2i-java/0.1/s2i-java.yaml +++ b/task/s2i-java/0.1/s2i-java.yaml @@ -273,17 +273,9 @@ spec: workingDir: $(workspaces.source.path) - name: upload-sbom - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting - # the cluster will set imagePullPolicy to IfNotPresent - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" workingDir: $(workspaces.source.path) volumes: diff --git a/task/s2i-nodejs/0.1/s2i-nodejs.yaml b/task/s2i-nodejs/0.1/s2i-nodejs.yaml index b526432be6..a44f2e9bfa 100644 --- a/task/s2i-nodejs/0.1/s2i-nodejs.yaml +++ b/task/s2i-nodejs/0.1/s2i-nodejs.yaml @@ -240,18 +240,10 @@ spec: workingDir: $(workspaces.source.path) - name: upload-sbom - image: quay.io/redhat-appstudio/cosign:v2.1.1@sha256:c883d6f8d39148f2cea71bff4622d196d89df3e510f36c140c097b932f0dd5d5 - # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting - # the cluster will set imagePullPolicy to IfNotPresent - args: - - attach - - sbom - - --sbom - - sbom-cyclonedx.json - - --type - - cyclonedx - - $(params.IMAGE) + image: quay.io/konflux-ci/appstudio-utils:ab6b0b8e40e440158e7288c73aff1cf83a2cc8a9@sha256:24179f0efd06c65d16868c2d7eb82573cce8e43533de6cea14fec3b7446e0b14 workingDir: $(workspaces.source.path) + script: | + cosign attach sbom --sbom sbom-cyclonedx.json --type cyclonedx "$(cat "$(results.IMAGE_REF.path)")" volumes: - emptyDir: {} From b8c448785512ca00bedd25b28b0a819ec0bd031f Mon Sep 17 00:00:00 2001 From: Tomas Nevrlka Date: Thu, 12 Sep 2024 18:02:13 +0200 Subject: [PATCH 3/4] docs: improve build-definitions README - Describe steps needed for local testing - Improve info about test-build.sh and test-builds.sh - Fix typos and other nitpicks --- README.md | 115 ++++++++++++++++++++++++++------------- task-generator/README.md | 0 2 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 task-generator/README.md diff --git a/README.md b/README.md index 7ec17d05b5..722ee68284 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,119 @@ # build-definitions -This repository contains components that are installed or managed by the managed CI and Build Team. +This repository contains components that are managed by the Konflux Build Team Green. -This includes default Pipelines and Tasks. You need to have bootstrapped a working appstudio configuration from (see `https://github.com/redhat-appstudio/infra-deployments`) for the dev of pipelines or new tasks. +This includes default Pipelines and Tasks. You need to have bootstrapped a working App Studio configuration from (see `https://github.com/redhat-appstudio/infra-deployments`) for the dev of pipelines or new tasks. -Pipelines and Tasks are delivered into App Studio via quay organization `konflux-ci/tekton-catalog`. -Pipelines are bundled and pushed into repositories prefixed with `pipeline-` and tagged with `$GIT_SHA` (tag will be updated with every change). -Tasks are bundled and pushed into repositories prefixed with `task-` and tagged with `$VERSION` where `VERSION` is the task version (tag is updated when the task file contains any change in the PR) +Pipelines and Tasks are delivered into App Studio via the quay organization `konflux-ci/tekton-catalog`. +Pipelines are bundled and pushed into repositories prefixed with `pipeline-` and tagged with `$GIT_SHA` (the tag will be updated with every change). +Tasks are bundled and pushed into repositories prefixed with `task-` and tagged with `$VERSION`, where `VERSION` is the task version (the tag is updated when the task file contains any change in the PR) -Currently a set of utilities are bundled with App Studio in `quay.io/konflux-ci/appstudio-utils:$GIT_SHA` as a convenience but tasks may be run from different per-task containers. +Currently, a set of utilities is bundled with App Studio in `quay.io/konflux-ci/appstudio-utils:$GIT_SHA` as a convenience, but tasks may be run from different per-task containers. ## Building -Script `hack/build-and-push.sh` creates bundles for pipelines, tasks and create appstudio-utils image. Images are pushed into your quay.io repository. You will need to set `QUAY_NAMESPACE` to use this feature and be logged into quay.io on your workstation. -Once you run the `hack/build-and-push.sh` all pipelines will come from your bundle instead of from the default installed by gitops into the cluster. +The script `hack/build-and-push.sh` creates bundles for pipelines, tasks and the `appstudio-utils` image. Images are pushed into your quay.io repository. You will need to set `QUAY_NAMESPACE` to use this feature and be logged into quay.io on your workstation. +Once you run the `hack/build-and-push.sh`, all pipelines will come from your bundle instead of the default one installed by GitOps into the cluster. > **Note** > -> If you're using Mac OS, you need to install [GNU coreutils](https://formulae.brew.sh/formula/coreutils) before running the `hack/build-and-push.sh` script: +> If you're using macOS, you need to install [GNU coreutils](https://formulae.brew.sh/formula/coreutils) before running the `hack/build-and-push.sh` script: > ```bash > brew install coreutils > ``` -There is an option to push all bundles to a single quay.io repository (this method is used in PR testing). It is used by setting `TEST_REPO_NAME` environment variable. Bundle names are then specified in the container image tag, i.e. `quay.io//$TEST_REPO_NAME:-` +There is an option to push all bundles to a single quay.io repository (this method is used in PR testing). It is used by setting a `TEST_REPO_NAME` environment variable. Bundle names are then specified in the container image tag, i.e., `quay.io//$TEST_REPO_NAME:-` ### Pipelines The pipelines can be found in the `pipelines` directory. -- `core-services`: contains pipeline for the CI of Stonesoup core services e.g. application-service and build-service. -- `template-build`: contains common template used to generate `docker-build`, `fbc-builder`, `java-builder` and `nodejs-builder` pipelines +- `core-services`: contains pipelines for the CI of Konflux core services e.g., `application-service` and `build-service`. +- `template-build`: contains common template used to generate `docker-build`, `fbc-builder`, `java-builder` and `nodejs-builder` pipelines. ### Tasks -The tasks can be found in the `tasks` directories. Tasks are bundled and used by bundled pipelines. Tasks are not stored in the Cluster. -For quick local innerloop style task development, you may install new Tasks in your local namespace manually and create your pipelines as well as the base task image to test new function. Tasks can be installed into local namespace using `oc apply -k tasks/appstudio-utils/util-tasks`. +The tasks can be found in the `tasks` directories. Tasks are bundled and used by bundled pipelines. Tasks are not stored in the cluster. +For quick local inner-loop-style task development, you may install new Tasks in your local namespace manually and create your pipelines, as well as the base task image, to test new functionality. Tasks can be installed into the local namespace using `oc apply -k tasks/appstudio-utils/util-tasks`. -There is a container which is used to support multiple set of tasks called `quay.io/konflux-ci/appstudio-utils:GIT_SHA` , which is a single container which is used by multiple tasks. Tasks may also be in their own container as well however many simple tasks are utilities and will be packaged for app studio in a single container. Tasks can rely on other tasks in the system which are co-packed in a container allowing combined tasks (build-only vs build-deploy) which use the same core implementations. +There is a container used to support multiple sets of tasks called `quay.io/konflux-ci/appstudio-utils:GIT_SHA`. This is a single container used by multiple tasks. Tasks may also be in their own containers as well. However, many simple tasks are utilities and will be packaged for App Studio in a single container. Tasks can rely on other tasks in the system, which are co-packed in a container, allowing combined tasks (build-only vs. build-deploy) that use the same core implementations. -Shellspec tests can be run by invoking `hack/test-shellspec.sh`. ### StepActions -The StepActions can be found in the `stepactions` directory. StepActions are not yet bundled. - -## Release - -Release is done by (better leave it to the [push pipeline](.tekton/push.yaml)): +Take a look at the [Tekton documentation](https://tekton.dev/docs/pipelines/stepactions/) for more information about StepActions. -```bash -for quay_namespace in redhat-appstudio-tekton-catalog konflux-ci/tekton-catalog; do - QUAY_NAMESPACE=$quay_namespace BUILD_TAG=$(git rev-parse HEAD) hack/build-and-push.sh -done -``` +The StepActions can be found in the `stepactions` directory. StepActions are not yet bundled. ### Versioning -When the task update changes the interface (eg. change of parameters, workspaces or results names) then a new version of the task should be created. The folder with the new version must contain `MIGRATION.md` with instructions on how to update the current pipeline file in user's `.tekton` folder. +When a task update changes the interface (e.g., change of parameters, workspaces or results names), a new version of the task should be created. The folder with the new version must contain `MIGRATION.md` with instructions on how to update the current pipeline file in user's `.tekton` folder. -Adding a new parameter with a default value does not require the task version increase. +Adding a new parameter with a default value does not require a task version increase. -Task version increase must be approved by Project Manager. +Task version increase must be approved by the Project Manager. -## Testing +## Local development +Tasks can have a TA (Trusted Artifact) version. +The recommended workflow is to only edit the base version and let the other versions get generated automatically. +``` +./hack/generate-ta-tasks.sh +``` +Buildah also has a remote version, which can be generated with: +``` +./hack/generate-buildah-remote.sh +``` -Script `./hack/test-builds.sh` creates pipelines and tasks directly in current namespace and executes there test builds. By setting the environment variable `QUAY_NAMESPACE` the images will be pushed into user's quay repository, in that case creation of secret named `redhat-appstudio-staginguser-pull-secret` is required. +## Testing -Script `./hack/test-build.sh` provides way to test on custom git repository and pipeline. Usage example: `./hack/test-build.sh https://github.com/jduimovich/spring-petclinic java-builder`. +### Prerequisites +- Provisioned cluster with sufficient resources +- Deployed Konflux on the cluster (see [infra-deployments](https://github.com/redhat-appstudio/infra-deployments)) + +1. Set up the image repository +PipelineRuns attempt to push to `registry.redhat.io` by default. +For testing, you will likely want to use your own Quay repository. +Specify the Quay repository using the `QUAY_NAMESPACE` environment variable in the format `OWNER/REPOSITORY_NAME`. +2. Set up the `redhat-appstudio-staginguser-pull-secret` + - Log in to `quay.io` using your credentials: + ``` + podman login quay.io + ``` + This will create an `auth.json` file in `${XDG_RUNTIME_DIR}/containers/auth.json`, which you will use to create a secret in the cluster. + - Create the pull secret in you cluster: + ``` + oc create secret docker-registry redhat-appstudio-staginguser-pull-secret --from-file=.dockerconfigjson=${XDG_RUNTIME_DIR}/containers/auth.json + ``` + - Link the secret to your service account: + ``` + oc secrets link appstudio-pipeline redhat-appstudio-staginguser-pull-secret + ``` +3. Run the tests + +- To test a custom Git repository and pipeline, use `./hack/test-build.sh`. + + Usage example: + ``` + ./hack/test-build.sh https://github.com/jduimovich/spring-petclinic java-builder`. + ``` +- To run tests on predefined Git repositories and pipelines, use: + ``` + ./hack/test-builds.sh + ``` +- Shellspec tests can be run by invoking: + ``` + ./hack/test-shellspec.sh` + ``` ### Compliance -Task definitions must comply to [Enterprise Contract](https://enterprisecontract.dev/) policies. -Currently, there are two policy configurations. The [all-tasks](./policies/all-tasks.yaml) policy -configuration applies to all Task definitions, while the [build-tasks](./policies/build-tasks.yaml) -policy configuration applies only to build Task definitions. A build Task, i.e. one that produces a -container image, must abide to both policy configurations. +Task definitions must comply with the [Enterprise Contract](https://enterprisecontract.dev/) policies. +Currently, there are two policy configurations. +- The [all-tasks](./policies/all-tasks.yaml) policy +configuration applies to all Task definitions +- The [build-tasks](./policies/build-tasks.yaml) +policy configuration applies only to build Task definitions. + +A build Task, i.e., one that produces a +container image, must abide by both policy configurations. diff --git a/task-generator/README.md b/task-generator/README.md new file mode 100644 index 0000000000..e69de29bb2 From 03b0bddf31fc07c846488ff4c16112a1c18b7353 Mon Sep 17 00:00:00 2001 From: Tomas Nevrlka Date: Fri, 13 Sep 2024 13:08:19 +0200 Subject: [PATCH 4/4] docs: add README for task-generator Task-generator doesn't have a README. Add a README with a description of what the generator does and its usage --- README.md | 18 ++++++++---------- task-generator/README.md | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 722ee68284..80bc7db9d1 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # build-definitions -This repository contains components that are managed by the Konflux Build Team Green. +This repository contains components that are managed by the Konflux Build Team. -This includes default Pipelines and Tasks. You need to have bootstrapped a working App Studio configuration from (see `https://github.com/redhat-appstudio/infra-deployments`) for the dev of pipelines or new tasks. +This includes default Pipelines and Tasks. You need to have bootstrapped a working Konflux configuration from (see `https://github.com/redhat-appstudio/infra-deployments`) for the dev of pipelines or new tasks. -Pipelines and Tasks are delivered into App Studio via the quay organization `konflux-ci/tekton-catalog`. +Pipelines and Tasks are delivered into Konflux via the quay organization `konflux-ci/tekton-catalog`. Pipelines are bundled and pushed into repositories prefixed with `pipeline-` and tagged with `$GIT_SHA` (the tag will be updated with every change). Tasks are bundled and pushed into repositories prefixed with `task-` and tagged with `$VERSION`, where `VERSION` is the task version (the tag is updated when the task file contains any change in the PR) -Currently, a set of utilities is bundled with App Studio in `quay.io/konflux-ci/appstudio-utils:$GIT_SHA` as a convenience, but tasks may be run from different per-task containers. +Currently, a set of utilities is bundled with Konflux in `quay.io/konflux-ci/appstudio-utils:$GIT_SHA` as a convenience, but tasks may be run from different per-task containers. ## Building @@ -37,7 +37,7 @@ The pipelines can be found in the `pipelines` directory. The tasks can be found in the `tasks` directories. Tasks are bundled and used by bundled pipelines. Tasks are not stored in the cluster. For quick local inner-loop-style task development, you may install new Tasks in your local namespace manually and create your pipelines, as well as the base task image, to test new functionality. Tasks can be installed into the local namespace using `oc apply -k tasks/appstudio-utils/util-tasks`. -There is a container used to support multiple sets of tasks called `quay.io/konflux-ci/appstudio-utils:GIT_SHA`. This is a single container used by multiple tasks. Tasks may also be in their own containers as well. However, many simple tasks are utilities and will be packaged for App Studio in a single container. Tasks can rely on other tasks in the system, which are co-packed in a container, allowing combined tasks (build-only vs. build-deploy) that use the same core implementations. +There is a container used to support multiple sets of tasks called `quay.io/konflux-ci/appstudio-utils:GIT_SHA`. This is a single container used by multiple tasks. Tasks may also be in their own containers as well. However, many simple tasks are utilities and will be packaged for Konflux in a single container. Tasks can rely on other tasks in the system, which are co-packed in a container, allowing combined tasks (build-only vs. build-deploy) that use the same core implementations. ### StepActions @@ -52,8 +52,6 @@ When a task update changes the interface (e.g., change of parameters, workspaces Adding a new parameter with a default value does not require a task version increase. -Task version increase must be approved by the Project Manager. - ## Local development Tasks can have a TA (Trusted Artifact) version. The recommended workflow is to only edit the base version and let the other versions get generated automatically. @@ -72,7 +70,7 @@ Buildah also has a remote version, which can be generated with: - Deployed Konflux on the cluster (see [infra-deployments](https://github.com/redhat-appstudio/infra-deployments)) 1. Set up the image repository -PipelineRuns attempt to push to `registry.redhat.io` by default. +PipelineRuns attempt to push to cluster-internal registry `image-registry.openshift-image-registry.svc:5000` by default. For testing, you will likely want to use your own Quay repository. Specify the Quay repository using the `QUAY_NAMESPACE` environment variable in the format `OWNER/REPOSITORY_NAME`. 2. Set up the `redhat-appstudio-staginguser-pull-secret` @@ -95,11 +93,11 @@ Specify the Quay repository using the `QUAY_NAMESPACE` environment variable in t Usage example: ``` - ./hack/test-build.sh https://github.com/jduimovich/spring-petclinic java-builder`. + QUAY_NAMESPACE=OWNER/REPOSITORY_NAME ./hack/test-build.sh https://github.com/jduimovich/spring-petclinic java-builder`. ``` - To run tests on predefined Git repositories and pipelines, use: ``` - ./hack/test-builds.sh + QUAY_NAMESPACE=OWNER/REPOSITORY_NAME ./hack/test-builds.sh ``` - Shellspec tests can be run by invoking: ``` diff --git a/task-generator/README.md b/task-generator/README.md index e69de29bb2..dd0d440291 100644 --- a/task-generator/README.md +++ b/task-generator/README.md @@ -0,0 +1,23 @@ +# Task generator +Task generator is a tool used for generating various versions of tasks. + +## Buildah remote task generation +The Buildah task builds source code into a container image and pushes the image into container registry using the Buildah tool. +While the standard Buildah task runs directly on the cluster, the remote task is run on a remote host. +This must be used in combination with the Multi Arch Controller, which provides the credentials and host name used to perform the build. + +The remote versions of the Buildah task are programmatically generated by this script from the buildah task to keep them in sync. The generated remote tasks should not be manually modified. + +### Arguments +- `buildah-task` - The location of the buildah task YAML file (required) +- `remote-task` - The location of the buildah-remote task YAML file to overwrite (required) +- `task-version` - The version of the task to overwrite, e.g. `0.2` (required) + +Example usage: +``` +go run remote/main.go \ + --buildah-task ../task/buildah/0.2/buildah.yaml \ + --remote-task ../task/buildah-remote/0.2/buildah-remote.yaml \ + --task-version 0.2 +``` +