diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml deleted file mode 100644 index f1974b03..00000000 --- a/.github/workflows/build-and-push.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Build and push container image - -env: - IMAGE_NAME: "rapidast" - IMAGE_TAGS: "${{ github.sha }}" - IMAGE_REGISTRY: quay.io/redhatproductsecurity - IMAGE_REGISTRY_USER: ${{ secrets.IMAGE_REGISTRY_USER }} - IMAGE_REGISTRY_PASSWORD: ${{ secrets.IMAGE_REGISTRY_PASSWORD }} - -on: - push: - branches: ["development", "main"] - -jobs: - - build-and-push: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - # https://github.com/redhat-actions/buildah-build#readme - - name: Build container image - id: build-image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - dockerfiles: | - ./containerize/Containerfile - - # https://github.com/redhat-actions/push-to-registry#readme - - name: Push to registry - id: push-image - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image.outputs.image }} - tags: ${{ steps.build-image.outputs.tags }} - registry: ${{ env.IMAGE_REGISTRY }} - username: ${{ env.IMAGE_REGISTRY_USER }} - password: ${{ env.IMAGE_REGISTRY_PASSWORD }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index e1ce7f9e..00000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build container image - -env: - IMAGE_NAME: "rapidast" - IMAGE_TAGS: "${{ github.sha }}" - -on: - pull_request: - branches: ["development", "main"] - -jobs: - - build-image: - - runs-on: ubuntu-latest - - # https://github.com/redhat-actions/buildah-build#readme - steps: - - uses: actions/checkout@v4 - - name: Build container image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - dockerfiles: | - ./containerize/Containerfile diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 96005c4c..c9f8e8a1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,8 +10,7 @@ permissions: contents: read jobs: - test: - + lint: runs-on: ubuntu-latest steps: @@ -24,9 +23,23 @@ jobs: run: | python3 -m ensurepip --upgrade pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt - - name: Test with pytest - run: | - pytest - name: Lint with pre-commit hook run: | pre-commit run --all-files --show-diff-on-failure + + unit-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: "3.9" + - name: Install dependencies + run: | + python3 -m ensurepip --upgrade + pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt + - name: Test with pytest + run: | + pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88488353..13608cee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: # W0603 - Using the global statement # C0114,C0115,C0116 - docstring checks. Disabled because of pydocstyle checks # W0107 - unnecessary pass + # W0511: fixme # W0702: No exception type(s) specified (bare-except) # R0801: Similar lines in 2 files. Disabled because it flags any file even those which are unrelated # R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return) @@ -51,6 +52,6 @@ repos: args: - --max-line-length=120 - --min-public-methods=0 - - --good-names=q,f,fp,i,e - - --disable=E0401,W1201,W1203,C0114,C0115,C0116,C0411,W0107,W0702,R0801,R1705,R1710 + - --good-names=w,q,f,fp,i,e + - --disable=E0401,W1201,W1203,C0114,C0115,C0116,C0411,W0107,W0511,W0702,R0801,R1705,R1710 language_version: python3 diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml new file mode 100644 index 00000000..1fcd1d05 --- /dev/null +++ b/.tekton/integration-test.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: rapidast-e2e +spec: + params: + - name: repo_url + default: github.com/RedHatProductSecurity/rapidast + - name: revision + default: "development" + - description: 'Snapshot of the application' + name: SNAPSHOT + default: |- + '{ + "components": [ + { + "name":"rapidast", + "containerImage": "quay.io/redhatproductsecurity/rapidast:latest", + "source":{ + "git":{ + "url":"git@github.com:RedHatProductSecurity/rapidast.git", + "revision":"development", + } + } + } + ] + }' + type: string + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: provision-eaas-space + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/provision-env-with-ephemeral-namespace/0.1/provision-env-with-ephemeral-namespace.yaml + params: + - name: KONFLUXNAMESPACE + value: $(context.pipelineRun.namespace) + - name: PIPELINERUN_NAME + value: $(context.pipelineRun.name) + - name: PIPELINERUN_UID + value: $(context.pipelineRun.uid) + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + # + # - name: clone-repository + # runAfter: + # - parse-metadata + # params: + # - name: url + # value: "$(tasks.parse-metadata.results.source-git-url)" + # - name: revision + # value: "$(tasks.parse-metadata.results.source-git-revision)" + # taskRef: + # name: git-clone + # workspaces: + # - name: output + # workspace: source + + - name: run-e2e-tests + runAfter: + - provision-eaas-space + taskSpec: + volumes: + - name: credentials + emptyDir: {} + results: + - name: TEST_RESULTS + description: e2e test results + steps: + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + - name: clone-repository + image: quay.io/konflux-ci/git-clone:latest + script: | + git config --global --add safe.directory /workspace + git clone "$(tasks.parse-metadata.results.source-git-url)" /workspace + pushd /workspace + git checkout "$(tasks.parse-metadata.results.source-git-revision)" + + - name: test + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: /tmp/kubeconfig + - name: KUBECONFIG_VALUE + valueFrom: + secretKeyRef: + name: $(tasks.provision-eaas-space.results.secretRef) + key: kubeconfig + - name: RAPIDAST_CLEANUP + value: "false" # namespace will be cleaned up automatically + - name: RAPIDAST_IMAGE + value: $(tasks.parse-metadata.results.component-container-image) + - name: RAPIDAST_SERVICEACCOUNT + value: namespace-manager # created by provision-env-with-ephemeral-namespace + workingDir: /workspace + volumeMounts: + - name: credentials + mountPath: /credentials + script: | + #!/bin/bash -ex + + echo "$KUBECONFIG_VALUE" > "$KUBECONFIG" + oc whoami + + yum install -y python3.12 + python3.12 -m ensurepip + pip3 install -r requirements.txt -r requirements-dev.txt + pytest -s e2e-tests --json-report --json-report-summary --json-report-file $(results.TEST_RESULTS.path) + cat $(results.TEST_RESULTS.path) diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml new file mode 100644 index 00000000..3e833400 --- /dev/null +++ b/.tekton/rapidast-pull-request.yaml @@ -0,0 +1,475 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RedHatProductSecurity/rapidast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && (target_branch == "development" || target_branch == "main") + creationTimestamp: null + labels: + appstudio.openshift.io/application: rapidast + appstudio.openshift.io/component: rapidast + pipelines.appstudio.openshift.io/type: build + name: rapidast-on-pull-request + namespace: secaut-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhatproductsecurity/rapidast:on-pr-{{revision}} + - name: image-expires-after + value: 7d + - name: dockerfile + value: containerize/Containerfile + - name: skip-checks + value: "true" + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:52f8b96b96ce4203d4b74d850a85f963125bf8eef0683ea5acdd80818d335a28 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:2a0c67ea7d5d82b4ec47930c12397f94b3af0b3855d8e5ad9f6e088c93e42bf0 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:327d745a58c1589b0ff196ed526d12a8a0a20ae22fd1c9dd1577b850a977dc3b + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:28fee4bf5da87f2388c973d9336086749cad8436003f9a514e22ac99735e056b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:7e99a122bc9e84fd9fb29062e825d3345177337d2448dcb50324f86ec5560c7a + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:a94b6523ba0b691dc276e37594321c2eff3594d2753014e5c920803b47627df1 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:e32feb2c815116730917fe5665d9f003e53f2e1718f60bcbabf0ab3abad5d7d4 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/rapidast-push.yaml b/.tekton/rapidast-push.yaml new file mode 100644 index 00000000..759c9e22 --- /dev/null +++ b/.tekton/rapidast-push.yaml @@ -0,0 +1,470 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RedHatProductSecurity/rapidast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && (target_branch == "development" || target_branch == "main") + creationTimestamp: null + labels: + appstudio.openshift.io/application: rapidast + appstudio.openshift.io/component: rapidast + pipelines.appstudio.openshift.io/type: build + name: rapidast-on-push + namespace: secaut-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhatproductsecurity/rapidast:{{revision}} + - name: dockerfile + value: containerize/Containerfile + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:52f8b96b96ce4203d4b74d850a85f963125bf8eef0683ea5acdd80818d335a28 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:2a0c67ea7d5d82b4ec47930c12397f94b3af0b3855d8e5ad9f6e088c93e42bf0 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:327d745a58c1589b0ff196ed526d12a8a0a20ae22fd1c9dd1577b850a977dc3b + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:28fee4bf5da87f2388c973d9336086749cad8436003f9a514e22ac99735e056b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:7e99a122bc9e84fd9fb29062e825d3345177337d2448dcb50324f86ec5560c7a + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:a94b6523ba0b691dc276e37594321c2eff3594d2753014e5c920803b47627df1 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:e32feb2c815116730917fe5665d9f003e53f2e1718f60bcbabf0ab3abad5d7d4 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml b/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml new file mode 100644 index 00000000..0648aecf --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "oobttest" + + general: + container: + # currently supported: `podman` and `none` + type: "none" + + scanners: + generic_1: + # results: + # An absolute path to file or directory where results are stored on the host. + # if it is "*stdout" or unspecified, the command's standard output will be selected + results: "/tmp/oobtkube.sarif.json" # if None or "*stdout", the command's standard output is selected + # toolDir: scanners/generic/tools + inline: "python3 oobtkube.py --log-level debug -d 60 -p 6000 -i rapidast-oobtkube -f /opt/rapidast/config/cr_example.yaml | tee /tmp/oobtkube.sarif.json" + # XXX using tekton Task because it has: + # - a resource type that has a .spec field (required by oobtkube script) + # - fields in .spec that are arbitrary string (.spec.description) + # this could be replaced later with something more common like a ConfigMap, + # once oobtkube can test non .spec values + cr_example.yaml: |+ + apiVersion: tekton.dev/v1 + kind: Task + metadata: + name: vulnerable + spec: + description: foobar + params: + - name: foo + type: string + steps: + - image: foo + name: foo + script: foo + +kind: ConfigMap +metadata: + name: rapidast-oobtkube diff --git a/e2e-tests/manifests/rapidast-oobtkube-pod.yaml b/e2e-tests/manifests/rapidast-oobtkube-pod.yaml new file mode 100644 index 00000000..bd344915 --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-pod.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-oobtkube + labels: + app: rapidast-oobtkube +spec: + containers: + - command: + - bash + - -c + - ./rapidast.py + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-oobtkube + resources: + limits: + cpu: 0.5 + memory: 1Gi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + serviceAccountName: ${SERVICEACCOUNT} # oobtkube needs perms to query API server + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-oobtkube + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never diff --git a/e2e-tests/manifests/rapidast-oobtkube-service.yaml b/e2e-tests/manifests/rapidast-oobtkube-service.yaml new file mode 100644 index 00000000..5cf4ccda --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: rapidast-oobtkube + name: rapidast-oobtkube +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 6000 + protocol: TCP + targetPort: 6000 + selector: + app: rapidast-oobtkube + sessionAffinity: None + type: ClusterIP diff --git a/e2e-tests/manifests/rapidast-trivy-configmap.yaml b/e2e-tests/manifests/rapidast-trivy-configmap.yaml new file mode 100644 index 00000000..f0fcdca3 --- /dev/null +++ b/e2e-tests/manifests/rapidast-trivy-configmap.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + # WARNING: `configVersion` indicates the schema version of the config file. + # This value tells RapiDAST what schema should be used to read this configuration. + # Therefore you should only change it if you update the configuration to a newer schema + # It is intended to keep backward compatibility (newer RapiDAST running an older config) + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "my-cluster" + + # `scanners' is a section that configures scanning options + scanners: + generic_trivy: + # results: + # An absolute path to file or directory where results are stored on the host. + # if it is "*stdout" or unspecified, the command's standard output will be selected + # When container.type is 'podman', this needs to be used along with the container.volumes configuration below + # If the result needs to be sent to DefectDojo, this must be a SARIF format file + #results: "/path/to/results" + + # Example: scan a k8s cluster for misconfiguration issue + # - See https://aquasecurity.github.io/trivy/v0.49/docs/target/kubernetes/ for more information on 'trivy k8s' scan + # - scanners/generic/tools/convert_trivy_k8s_to_sarif.py converts the Trivy json result to the SARIF format + # 'inline' is used when container.type is not 'podman' + # 'toolDir' specifies the default directory where inline scripts are located + #toolDir: scanners/generic/tools + inline: "trivy k8s -n $(cat /run/secrets/kubernetes.io/serviceaccount/namespace) pod --scanners=misconfig --report all --format json --skip-policy-update | convert_trivy_k8s_to_sarif.py" + + container: + parameters: + # Optional: list of expected return codes, anything else will be considered as an error. by default: [0] + validReturns: [ 0 ] + +kind: ConfigMap +metadata: + name: rapidast-trivy diff --git a/e2e-tests/manifests/rapidast-trivy-pod.yaml b/e2e-tests/manifests/rapidast-trivy-pod.yaml new file mode 100644 index 00000000..e82f4fdf --- /dev/null +++ b/e2e-tests/manifests/rapidast-trivy-pod.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-trivy +spec: + containers: + - command: + - bash + - -c + - ./rapidast.py + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-trivy + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 125m + memory: 256Mi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + serviceAccountName: ${SERVICEACCOUNT} # trivy needs perms to query API server + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-trivy + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never diff --git a/e2e-tests/manifests/rapidast-vapi-configmap.yaml b/e2e-tests/manifests/rapidast-vapi-configmap.yaml new file mode 100644 index 00000000..d5e0b9a2 --- /dev/null +++ b/e2e-tests/manifests/rapidast-vapi-configmap.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "v5-none-release-test" + url: "http://vapi:5000" + + scanners: + zap: + # define a scan through the ZAP scanner + apiScan: + apis: + apiUrl: "http://vapi:5000/docs/openapi.json" + + passiveScan: + # optional list of passive rules to disable + disabledRules: "2,10015,10027,10096,10024,10098,10023,10105" + + activeScan: + policy: API-scan-minimal + + container: + parameters: + executable: "zap.sh" + + miscOptions: + # enableUI (default: false), requires a compatible runtime (e.g.: flatpak or no containment) + #enableUI: True + # Defaults to True, set False to prevent auto update of ZAP plugins + updateAddons: False + # additionalAddons: ascanrulesBeta + # If set to True and authentication is oauth2_rtoken and api.apiUrl is set, download the API outside of ZAP + oauth2OpenapiManualDownload: False + +kind: ConfigMap +metadata: + name: rapidast-vapi diff --git a/e2e-tests/manifests/rapidast-vapi-pod.yaml b/e2e-tests/manifests/rapidast-vapi-pod.yaml new file mode 100644 index 00000000..a5d7a957 --- /dev/null +++ b/e2e-tests/manifests/rapidast-vapi-pod.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-vapi +spec: + containers: + - command: + - bash + - -c + - "./rapidast.py && cat results/*/*/zap/zap-report.json" # ugly, but saves needing a PVC to retrieve .json file after execution + # - "./rapidast.py --log-level debug && sleep infinity" # keep the pod alive so we can exec into it to parse reports + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-vapi + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 250m + memory: 512Mi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + # - name: results-volume + # mountPath: /opt/rapidast/results + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-vapi + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never +# --- +# apiVersion: v1 +# kind: PersistentVolumeClaim +# metadata: +# name: rapidast-results +# spec: +# accessModes: +# - ReadWriteOnce +# volumeMode: Filesystem +# resources: +# requests: +# storage: 1Gi diff --git a/e2e-tests/manifests/task-controller-deployment.yaml b/e2e-tests/manifests/task-controller-deployment.yaml new file mode 100644 index 00000000..dc8e4266 --- /dev/null +++ b/e2e-tests/manifests/task-controller-deployment.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + name: task-controller + labels: + app: task-controller +spec: + replicas: 1 + selector: + matchLabels: + app: task-controller + template: + metadata: + labels: + app: task-controller + spec: + containers: + # simulates a custom controller that monitors for tekton Tasks named "vulnerable" + # and tries to execute a deliberate command injection + # Tasks are chosen only because they have a field in .spec that can be arbitrary string + - command: + - bash + - -c + - | + while true; do + sleep 2 + sh -c "$(oc get task/vulnerable -o=jsonpath='{.spec.description}')" + done + image: registry.redhat.io/openshift4/ose-cli:latest + imagePullPolicy: Always + name: task-controller + serviceAccountName: ${SERVICEACCOUNT} # required to read Tasks from API server diff --git a/e2e-tests/manifests/vapi-deployment.yaml b/e2e-tests/manifests/vapi-deployment.yaml new file mode 100644 index 00000000..7b41f204 --- /dev/null +++ b/e2e-tests/manifests/vapi-deployment.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + name: vapi + labels: + app: vapi +spec: + replicas: 1 + selector: + matchLabels: + app: vapi + template: + metadata: + labels: + app: vapi + spec: + containers: + # command is custom because: + # - default entrypoint runs frontend and based on rapidast-consolidated-e2e.sh we only want backend + # - running start_backend.sh on its own is not sufficient, a sleep is needed to keep the pod alive + - command: + - bash + - -c + - . start_backend.sh && sleep infinity + image: quay.io/sfowler/vapi:latest + imagePullPolicy: Always + name: vapi diff --git a/e2e-tests/manifests/vapi-service.yaml b/e2e-tests/manifests/vapi-service.yaml new file mode 100644 index 00000000..fe64da8b --- /dev/null +++ b/e2e-tests/manifests/vapi-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: vapi + name: vapi +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 5000 + protocol: TCP + targetPort: 5000 + selector: + app: vapi + sessionAffinity: None + type: ClusterIP diff --git a/e2e-tests/test_integration.py b/e2e-tests/test_integration.py new file mode 100644 index 00000000..cd8126a1 --- /dev/null +++ b/e2e-tests/test_integration.py @@ -0,0 +1,201 @@ +import json +import os +import re +import shutil +import tempfile + +import certifi +import pytest +from kubernetes import client +from kubernetes import config +from kubernetes import utils +from kubernetes import watch +from kubernetes.client.rest import ApiException + +NAMESPACE = os.getenv("RAPIDAST_NAMESPACE", "") # e.g. rapidast--pipeline +SERVICEACCOUNT = os.getenv("RAPIDAST_SERVICEACCOUNT", "pipeline") # name of ServiceAccount used in rapidast pod +RAPIDAST_IMAGE = os.getenv("RAPIDAST_IMAGE", "quay.io/redhatproductsecurity/rapidast:latest") +# delete resources created by tests +RAPIDAST_CLEANUP = os.getenv("RAPIDAST_CLEANUP", "True").lower() in ("true", "1", "t", "y", "yes") + +MANIFESTS = "e2e-tests/manifests" + + +# monkeypatch certifi so that internal CAs are trusted +def where(): + return os.getenv("REQUESTS_CA_BUNDLE", "/etc/pki/tls/certs/ca-bundle.crt") + + +certifi.where = where + + +@pytest.fixture(name="kclient") +def fixture_kclient(): + config.load_config() + yield client.ApiClient() + + +def wait_until_ready(**kwargs): + corev1 = client.CoreV1Api() + w = watch.Watch() + for event in w.stream(func=corev1.list_namespaced_pod, namespace=NAMESPACE, timeout_seconds=60, **kwargs): + if not isinstance(event, dict): # Watch.stream() can yield non-dict types + continue + print(event["object"].metadata.name, event["object"].status.phase) + if event["object"].status.phase == "Running": + return + raise RuntimeError("Timeout out waiting for pod matching: {kwargs}") + + +# simulates: $ oc logs -f | tee +def tee_log(pod_name: str, filename: str): + corev1 = client.CoreV1Api() + w = watch.Watch() + with open(filename, "w", encoding="utf-8") as f: + for e in w.stream(corev1.read_namespaced_pod_log, name=pod_name, namespace=NAMESPACE): + if not isinstance(e, str): + continue # Watch.stream() can yield non-string types + f.write(e + "\n") + print(e) + + +def render_manifests(input_dir, output_dir): + shutil.copytree(input_dir, output_dir, dirs_exist_ok=True) + print(f"rendering manifests in {output_dir}") + print(f"using serviceaccount {SERVICEACCOUNT}") + # XXX should probably replace this with something like kustomize + for filepath in os.scandir(output_dir): + with open(filepath, "r", encoding="utf-8") as f: + contents = f.read() + contents = contents.replace("${IMAGE}", RAPIDAST_IMAGE) + contents = contents.replace("${SERVICEACCOUNT}", SERVICEACCOUNT) + with open(filepath, "w", encoding="utf-8") as f: + f.write(contents) + + +def setup_namespace(): + global NAMESPACE # pylint: disable=W0603 + # only try to create a namespace if env is set + if NAMESPACE == "": + NAMESPACE = get_current_namespace() + else: + create_namespace(NAMESPACE) + print(f"using namespace '{NAMESPACE}'") + + +def get_current_namespace() -> str: + try: + # Load the kubeconfig + config.load_config() + + # Get the kube config object + _, active_context = config.list_kube_config_contexts() + + # Return the namespace from current context + if active_context and "namespace" in active_context["context"]: + return active_context["context"]["namespace"] + return "default" + + except config.config_exception.ConfigException: + # If running inside a pod + try: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8") as f: + return f.read().strip() + except FileNotFoundError: + return "default" + + +def create_namespace(namespace_name: str): + config.load_config() + corev1 = client.CoreV1Api() + try: + corev1.read_namespace(namespace_name) + print(f"namespace {namespace_name} already exists") + except ApiException as e: + if e.status == 404: + print(f"creating namespace {namespace_name}") + namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace_name)) + corev1.create_namespace(namespace) + else: + raise e + except Exception as e: # pylint: disable=W0718 + print(f"error reading namespace {namespace_name}: {e}") + + +def cleanup(): + if RAPIDAST_CLEANUP: + os.system(f"kubectl delete -f {MANIFESTS}/") + # XXX oobtukbe does not clean up after itself + os.system("kubectl delete Task/vulnerable") + + +class TestRapiDAST: + @classmethod + def setup_class(cls): + cls.tempdir = tempfile.mkdtemp() + render_manifests(MANIFESTS, cls.tempdir) + print(f"testing with image: {RAPIDAST_IMAGE}") + setup_namespace() + cleanup() + + @classmethod + def teardown_class(cls): + # TODO teardown should really occur after each test, so the the + # resource count does not grown until quota reached + cleanup() + + def create_from_yaml(self, kclient, path: str): + # simple wrapper to reduce repetition + utils.create_from_yaml(kclient, path, namespace=NAMESPACE, verbose=True) + + def test_vapi(self, kclient): + """Test rapidast find expected number of findings in VAPI""" + self.create_from_yaml(kclient, f"{self.tempdir}/vapi-deployment.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/vapi-service.yaml") + wait_until_ready(label_selector="app=vapi") + + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-vapi-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-vapi-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-vapi") + + logfile = os.path.join(self.tempdir, "rapidast-vapi.log") + tee_log("rapidast-vapi", logfile) + + # XXX relies on rapidast-vapi pod cat-ing the result json file after execution + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + pattern = r"^{\s*$.*$" + matches = re.findall(pattern, logs, re.MULTILINE | re.DOTALL) + assert matches, f"{logfile} did not contain expected json results" + results = json.loads(matches[0]) + + assert len(results["site"][0]["alerts"]) == 3 + + def test_trivy(self, kclient): + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-trivy-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-trivy-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-trivy") + + logfile = os.path.join(self.tempdir, "rapidast-trivy.log") + tee_log("rapidast-trivy", logfile) + + expected_line = "INFO:scanner: 'generic_trivy' completed successfully" + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + assert expected_line in logs, f"{logfile} does not contain expected line: {expected_line}" + + def test_oobtkube(self, kclient): + self.create_from_yaml(kclient, f"{self.tempdir}/task-controller-deployment.yaml") + + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-service.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-oobtkube") + + logfile = os.path.join(self.tempdir, "rapidast-oobtkube.log") + tee_log("rapidast-oobtkube", logfile) + + expected_line = "RESULT: OOB REQUEST DETECTED" + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + assert expected_line in logs, f"{logfile} does not contain expected line: {expected_line}" diff --git a/requirements-dev.txt b/requirements-dev.txt index 3993dc3e..20e11a9f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,5 @@ pytest >= 7.2.1 black requests pre-commit == 3.7.1 +kubernetes +pytest-json-report