From 75e0df1e6a4ce2efc7ff1ead77435f1c4f13490d Mon Sep 17 00:00:00 2001 From: Luca Sartori Date: Tue, 22 Oct 2024 20:42:30 +0200 Subject: [PATCH 1/3] chore(ZCH-127): change_detection for django, dotnet, node, python, springboot, sonar-dotnet --- .github/workflows/django-workflow-common.yml | 56 ++++++++++++++-- .github/workflows/dotnet-workflow-common.yml | 53 +++++++++++++-- .github/workflows/node-workflow-common.yml | 57 +++++++++++++++-- .github/workflows/python-workflow-common.yml | 52 +++++++++++++-- .../workflows/sonar-step-dotnet-analyze.yml | 45 +++++++++++++ .../workflows/springboot-workflow-common.yml | 64 +++++++++++++++---- .prettierrc | 4 ++ docs/CHANGE_DETECTION.md | 58 +++++++++++++++++ package-lock.json | 18 ++++++ package.json | 7 +- 10 files changed, 380 insertions(+), 34 deletions(-) create mode 100644 .prettierrc create mode 100644 docs/CHANGE_DETECTION.md diff --git a/.github/workflows/django-workflow-common.yml b/.github/workflows/django-workflow-common.yml index a5ce5f71..43b2a9e9 100644 --- a/.github/workflows/django-workflow-common.yml +++ b/.github/workflows/django-workflow-common.yml @@ -22,11 +22,11 @@ on: RUN_ON: required: false type: string - default: 'zupit-agents' + default: "zupit-agents" RUNNERS_CONTAINER_GROUP: required: false type: string - default: 'Container' + default: "Container" RUN: required: false type: boolean @@ -48,11 +48,46 @@ on: required: false type: number default: 50 + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + django-lint-check: - uses: - ./.github/workflows/django-step-lint-check.yml + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + uses: ./.github/workflows/django-step-lint-check.yml with: RUN_ON: ${{inputs.RUN_ON}} RUNNERS_CONTAINER_GROUP: ${{inputs.RUNNERS_CONTAINER_GROUP}} @@ -66,8 +101,9 @@ jobs: secrets: inherit django-tests: - uses: - ./.github/workflows/django-step-tests.yml + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + uses: ./.github/workflows/django-step-tests.yml with: RUN_ON: ${{inputs.RUN_ON}} RUNNERS_CONTAINER_GROUP: ${{inputs.RUNNERS_CONTAINER_GROUP}} @@ -80,3 +116,11 @@ jobs: LFS_REPO_PATH: ${{inputs.LFS_REPO_PATH}} COVERAGE_THRESHOLD: ${{inputs.COVERAGE_THRESHOLD}} secrets: inherit + + jobs-succeded: + needs: ["django-lint-check", "django-tests"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: django-lint-check, django-tests didn't fail." + run: if [[ "${{ needs.django-lint-check.result }}" == "failure" || "${{ needs.django-tests.result }}" == "failure" ]]; then exit 1; fi diff --git a/.github/workflows/dotnet-workflow-common.yml b/.github/workflows/dotnet-workflow-common.yml index 64502df1..d77e851f 100644 --- a/.github/workflows/dotnet-workflow-common.yml +++ b/.github/workflows/dotnet-workflow-common.yml @@ -12,19 +12,19 @@ on: RUN_ON: required: false type: string - default: 'zupit-agents' + default: "zupit-agents" RUNNERS_CONTAINER_GROUP: required: false type: string - default: 'Container' + default: "Container" DOTNET_IMAGE_ENV_VARIABLES: required: false type: string - default: '{}' + default: "{}" CSHARPIER_VERSION: required: false type: string - default: '' + default: "" RUN_LINT: required: false type: boolean @@ -33,10 +33,47 @@ on: required: false type: boolean default: true + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + dotnet-common: + needs: workdir-has-changes name: Run .NET build, check formatting and test + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} runs-on: labels: ${{ inputs.RUN_ON }} group: ${{ inputs.RUNNERS_CONTAINER_GROUP }} @@ -76,3 +113,11 @@ jobs: with: WORKING_DIRECTORY: ${{ inputs.WORKING_DIRECTORY }} GENERATE_CODE_COVERAGE: false + + jobs-succeded: + needs: dotnet-common + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: dotnet-common didn't fail." + run: if [ "${{ needs.dotnet-common.result }}" == "failure" ]; then exit 1; fi diff --git a/.github/workflows/node-workflow-common.yml b/.github/workflows/node-workflow-common.yml index 23540b2c..64d3d393 100644 --- a/.github/workflows/node-workflow-common.yml +++ b/.github/workflows/node-workflow-common.yml @@ -29,11 +29,11 @@ on: RUN_ON: required: false type: string - default: 'zupit-agents' + default: "zupit-agents" RUNNERS_CONTAINER_GROUP: required: false type: string - default: 'Container' + default: "Container" RUN: required: false type: boolean @@ -42,11 +42,47 @@ on: required: false type: string default: "" + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + lint-check-build: - uses: - ./.github/workflows/node-step-format-lint-build.yml + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + uses: ./.github/workflows/node-step-format-lint-build.yml with: RUN_ON: ${{inputs.RUN_ON}} RUNNERS_CONTAINER_GROUP: ${{inputs.RUNNERS_CONTAINER_GROUP}} @@ -57,9 +93,9 @@ jobs: secrets: inherit cypress-run: - if: ${{ inputs.ENABLE_TESTS }} - uses: - ./.github/workflows/node-step-test-cypress.yml + needs: workdir-has-changes + if: ${{ inputs.ENABLE_TESTS && (!inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))) }} + uses: ./.github/workflows/node-step-test-cypress.yml with: RUN_ON: ${{inputs.RUN_ON}} RUNNERS_CONTAINER_GROUP: ${{inputs.RUNNERS_CONTAINER_GROUP}} @@ -71,3 +107,10 @@ jobs: PROJECT: ${{ inputs.PROJECT }} secrets: inherit + jobs-succeded: + needs: ["lint-check-build", "cypress-run"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: lint-check-build, cypress-run didn't fail." + run: if [[ "${{ needs.lint-check-build.result }}" == "failure" || "${{ needs.cypress-run.result }}" == "failure" ]]; then exit 1; fi diff --git a/.github/workflows/python-workflow-common.yml b/.github/workflows/python-workflow-common.yml index 5deaf5a6..65b0d012 100644 --- a/.github/workflows/python-workflow-common.yml +++ b/.github/workflows/python-workflow-common.yml @@ -18,11 +18,11 @@ on: RUN_ON: required: false type: string - default: 'zupit-agents' + default: "zupit-agents" RUNNERS_CONTAINER_GROUP: required: false type: string - default: 'Container' + default: "Container" RUN: required: false type: boolean @@ -36,11 +36,47 @@ on: required: false type: string default: "" + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + django-lint-check: - uses: - ./.github/workflows/python-step-lint-check.yml + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + uses: ./.github/workflows/python-step-lint-check.yml with: RUN_ON: ${{inputs.RUN_ON}} RUNNERS_CONTAINER_GROUP: ${{inputs.RUNNERS_CONTAINER_GROUP}} @@ -51,3 +87,11 @@ jobs: ENABLE_LFS: ${{inputs.ENABLE_LFS}} LFS_REPO_PATH: ${{inputs.LFS_REPO_PATH}} secrets: inherit + + jobs-succeded: + needs: ["django-lint-check"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: django-lint-check didn't fail." + run: if [[ "${{ needs.django-lint-check.result }}" == "failure" ]]; then exit 1; fi diff --git a/.github/workflows/sonar-step-dotnet-analyze.yml b/.github/workflows/sonar-step-dotnet-analyze.yml index 9a6a5fd7..2e35bfda 100644 --- a/.github/workflows/sonar-step-dotnet-analyze.yml +++ b/.github/workflows/sonar-step-dotnet-analyze.yml @@ -49,9 +49,46 @@ on: required: false type: string default: "**/*.trx" + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + sonar-analyze: + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job))) }} runs-on: labels: ${{ inputs.RUN_ON }} group: ${{ inputs.RUNNERS_CONTAINER_GROUP }} @@ -111,3 +148,11 @@ jobs: - name: End .NET SonarScanner run: dotnet-sonarscanner end /d:sonar.token="${{ secrets.SONAR_TOKEN }}" + + jobs-succeded: + needs: sonar-analyze + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: sonar-analyze didn't fail." + run: if [ "${{ needs.sonar-analyze.result }}" == "failure" ]; then exit 1; fi diff --git a/.github/workflows/springboot-workflow-common.yml b/.github/workflows/springboot-workflow-common.yml index a3e2282b..3682df9f 100644 --- a/.github/workflows/springboot-workflow-common.yml +++ b/.github/workflows/springboot-workflow-common.yml @@ -32,15 +32,49 @@ on: DATABASE: required: false type: string - description: 'Database to use: postgres or mysql' - default: 'postgres' + description: "Database to use: postgres or mysql" + default: "postgres" + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" -env: "${{secrets}}" +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + springboot-lint-check: - uses: - ./.github/workflows/springboot-step-lint-check.yml + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + uses: ./.github/workflows/springboot-step-lint-check.yml with: JAVA_IMAGE: ${{inputs.JAVA_IMAGE}} WORKING_DIRECTORY: ${{inputs.WORKING_DIRECTORY}} @@ -50,9 +84,9 @@ jobs: secrets: inherit springboot-tests: - if: ${{ inputs.DATABASE == 'postgres'}} - uses: - ./.github/workflows/springboot-step-tests.yml + needs: workdir-has-changes + if: ${{ inputs.DATABASE == 'postgres' && (!inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job))))}} + uses: ./.github/workflows/springboot-step-tests.yml with: JAVA_IMAGE: ${{inputs.JAVA_IMAGE}} WORKING_DIRECTORY: ${{inputs.WORKING_DIRECTORY}} @@ -64,9 +98,9 @@ jobs: secrets: inherit springboot-tests-mysql: - if: ${{ inputs.DATABASE == 'mysql'}} - uses: - ./.github/workflows/springboot-step-tests-mysql.yml + needs: workdir-has-changes + if: ${{ inputs.DATABASE == 'mysql' && (!inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job))))}} + uses: ./.github/workflows/springboot-step-tests-mysql.yml with: JAVA_IMAGE: ${{inputs.JAVA_IMAGE}} WORKING_DIRECTORY: ${{inputs.WORKING_DIRECTORY}} @@ -75,3 +109,11 @@ jobs: COVERAGE_ARTIFACT_NAME: ${{ inputs.COVERAGE_ARTIFACT_NAME }} RUN: ${{ inputs.RUN }} secrets: inherit + + jobs-succeded: + needs: ["springboot-lint-check", "springboot-tests", "springboot-tests-mysql"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: springboot-lint-check, springboot-tests, springboot-tests-mysql didn't fail." + run: if [[ "${{ needs.springboot-lint-check.result }}" == "failure" || "${{ needs.springboot-tests.result }}" == "failure" || "${{ needs.springboot-tests-mysql.result }}" == "failure" ]]; then exit 1; fi diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..b27104bb --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "singleQuote": false, + "printWidth": 120 +} diff --git a/docs/CHANGE_DETECTION.md b/docs/CHANGE_DETECTION.md new file mode 100644 index 00000000..021a9bbe --- /dev/null +++ b/docs/CHANGE_DETECTION.md @@ -0,0 +1,58 @@ +# Change detection based workflows + +## To use them on reusable workflows: + +_Reusable workflows might have different parameters. Refer to the specific workflows input variables and steps_ + +### Generic approach + +Add the parameters `CHECK_` parameters configuration to the workflow. + +- `CHECK_WORKDIR_CHANGES`: General toggle for the feature. When off it ignores workdir changes detection and always runs the next jobs + +- `CHECK_CUSTOM_DIR`: Specifies a custom DIR to check (use as override dir when the dir to check is differnt from the `WORKING_DIRECTORY`) + +- `CHECK_CHANGES_BY_JOBS`: String array that specifies the job id where this dir check is ran. Default is `'all'`. (ex: `"['pr-backend']"`) + +## To implement and use them on custom/local workflows: + +1. Add a job (ex: `workdir-has-changes`) that checks for changes on specific directories and outputs a variable (ex: `run-backend`) which stores `true` if it has detected changes or `false` if it didn't + +2. Add a job (ex: `jobs-succeded`) that fails when the child steps fails and succeeds when child jobs succeed or get skipped. + This will be the job that will be set on the branch protection policy + +3. Add a `needs` statement that refers to the check-changes job (ex: `workdir-has-changes`) + +4. Add an `if` statement that refers to the changes output variable (ex: `needs.workdir-has-changes.outputs.run-backend`) and skips the job if it evaluates to false + +### Full example + +```yml +jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + run-backend: ${{ steps.filter.outputs.backend }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + backend: + - 'Backend/**' + + pr-backend: + needs: workdir-has-changes + if: ${{ needs.workdir-has-changes.outputs.run-backend == 'true' }} + uses: ./.github/workflows/common-backend.yml + secrets: inherit + + jobs-succeded: + needs: ["pr-backend"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: pr-backend didn't fail." + run: if [[ "${{ needs.pr-backend.result }}" == "failure" ]]; then exit 1; fi +``` diff --git a/package-lock.json b/package-lock.json index eabb5302..f6e46ee7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "pipeline-templates", "version": "0.0.0-development", "license": "ISC", + "dependencies": { + "prettier": "^3.3.3" + }, "devDependencies": { "@commitlint/cli": "^17.6.5", "@commitlint/config-conventional": "^17.6.5", @@ -7591,6 +7594,21 @@ "node": ">=4" } }, + "node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "dev": true, diff --git a/package.json b/package.json index 01d8f19f..95fb159d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "semantic-release": "^21.0.3" }, "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "format": "prettier --write .", "semantic-release": "semantic-release" }, "repository": { @@ -34,5 +34,8 @@ "bugs": { "url": "https://github.com/zupit-it/pipeline-templates/issues" }, - "homepage": "https://github.com/zupit-it/pipeline-templates#readme" + "homepage": "https://github.com/zupit-it/pipeline-templates#readme", + "dependencies": { + "prettier": "^3.3.3" + } } From e79ef1212813dd10de38c59ff834d3c01f50b5c8 Mon Sep 17 00:00:00 2001 From: Luca Sartori Date: Sun, 17 Nov 2024 21:01:38 +0100 Subject: [PATCH 2/3] chore(ZCH-127): added missing sonar change_detections --- .github/workflows/sonar-step-analyze.yml | 43 ++++++++++++++++++- .../workflows/sonar-step-flutter-analyze.yml | 43 +++++++++++++++++++ .../sonar-step-springboot-analyze.yml | 43 +++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sonar-step-analyze.yml b/.github/workflows/sonar-step-analyze.yml index 212caa86..0d5456ea 100644 --- a/.github/workflows/sonar-step-analyze.yml +++ b/.github/workflows/sonar-step-analyze.yml @@ -34,13 +34,46 @@ on: required: false type: string default: ".coverage-reports/" - + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" sonar-analyze: + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} runs-on: labels: ${{ inputs.RUN_ON }} group: ${{ inputs.RUNNERS_CONTAINER_GROUP }} @@ -68,3 +101,11 @@ jobs: ARTIFACT_FILENAME: ${{ inputs.ARTIFACT_FILENAME }} ARTIFACT_PATH: ${{ inputs.ARTIFACT_PATH }} env: "${{secrets}}" + + jobs-succeded: + needs: ["sonar-analyze"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: sonar-analyze didn't fail." + run: if [[ "${{ needs.sonar-analyze.result }}" == "failure" ]]; then exit 1; fi diff --git a/.github/workflows/sonar-step-flutter-analyze.yml b/.github/workflows/sonar-step-flutter-analyze.yml index 4b8f1e0c..cd6700fa 100644 --- a/.github/workflows/sonar-step-flutter-analyze.yml +++ b/.github/workflows/sonar-step-flutter-analyze.yml @@ -38,13 +38,48 @@ on: required: false type: boolean default: false + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + sonar-analyze: + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} runs-on: labels: ${{ inputs.RUN_ON }} group: ${{ inputs.RUNNERS_CONTAINER_GROUP }} @@ -71,3 +106,11 @@ jobs: - name: Run Sonar run: sonar-scanner -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} -Dsonar.login=${{ secrets.SONAR_TOKEN }} -Dsonar.qualitygate.wait=${{ inputs.CHECK_QUALITY_GATE }} + + jobs-succeded: + needs: ["sonar-analyze"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: sonar-analyze didn't fail." + run: if [[ "${{ needs.sonar-analyze.result }}" == "failure" ]]; then exit 1; fi diff --git a/.github/workflows/sonar-step-springboot-analyze.yml b/.github/workflows/sonar-step-springboot-analyze.yml index 8866ec57..4798bd97 100644 --- a/.github/workflows/sonar-step-springboot-analyze.yml +++ b/.github/workflows/sonar-step-springboot-analyze.yml @@ -45,13 +45,48 @@ on: required: false type: string default: "" + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} jobs: + workdir-has-changes: + runs-on: ubuntu-latest + outputs: + changes-detected: ${{ steps.filter.outputs.changes-detected }} + steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + changes-detected: + - "${{ env.CHECK_DIR }}/**" + sonar-analyze: + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} runs-on: labels: ${{ inputs.RUN_ON }} group: ${{ inputs.RUNNERS_CONTAINER_GROUP }} @@ -76,3 +111,11 @@ jobs: - name: Run Sonar run: ./mvnw -ntp initialize sonar:sonar -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} -Dsonar.login=${{ secrets.SONAR_TOKEN }} -Dsonar.qualitygate.wait=${{ inputs.CHECK_QUALITY_GATE }} + + jobs-succeded: + needs: ["sonar-analyze"] + runs-on: ubuntu-latest + if: ${{ always()}} + steps: + - name: "Jobs: sonar-analyze didn't fail." + run: if [[ "${{ needs.sonar-analyze.result }}" == "failure" ]]; then exit 1; fi From 6230ea636fa795a5f1c5c05123dfe97740585540 Mon Sep 17 00:00:00 2001 From: Luca Sartori Date: Sun, 17 Nov 2024 21:05:18 +0100 Subject: [PATCH 3/3] chore(ZCH-127): better md --- docs/CHANGE_DETECTION.md | 189 +++++++++++++++++++++++++++++++++------ 1 file changed, 162 insertions(+), 27 deletions(-) diff --git a/docs/CHANGE_DETECTION.md b/docs/CHANGE_DETECTION.md index 021a9bbe..84c65f9a 100644 --- a/docs/CHANGE_DETECTION.md +++ b/docs/CHANGE_DETECTION.md @@ -1,58 +1,193 @@ -# Change detection based workflows +# Change Detection Pattern for GitHub Workflows -## To use them on reusable workflows: +This pattern allows workflows to run only when relevant changes are detected in specific directories. This optimization helps reduce unnecessary workflow runs and saves CI/CD resources. -_Reusable workflows might have different parameters. Refer to the specific workflows input variables and steps_ +## Overview -### Generic approach +The pattern consists of three main components: -Add the parameters `CHECK_` parameters configuration to the workflow. +1. A change detection job that checks for modifications in specified directories +2. Conditional execution of workflow jobs based on the detection results +3. A job status validation step that runs even if jobs are skipped -- `CHECK_WORKDIR_CHANGES`: General toggle for the feature. When off it ignores workdir changes detection and always runs the next jobs +## Implementation Guide -- `CHECK_CUSTOM_DIR`: Specifies a custom DIR to check (use as override dir when the dir to check is differnt from the `WORKING_DIRECTORY`) +### 1. Add Required Input Parameters -- `CHECK_CHANGES_BY_JOBS`: String array that specifies the job id where this dir check is ran. Default is `'all'`. (ex: `"['pr-backend']"`) +Add these parameters to your workflow's `workflow_call` inputs: -## To implement and use them on custom/local workflows: +```yaml +inputs: + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + description: "When true, enables change detection. When false, always runs jobs" -1. Add a job (ex: `workdir-has-changes`) that checks for changes on specific directories and outputs a variable (ex: `run-backend`) which stores `true` if it has detected changes or `false` if it didn't + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + description: "Override directory to check for changes. Defaults to WORKING_DIRECTORY if empty" -2. Add a job (ex: `jobs-succeded`) that fails when the child steps fails and succeeds when child jobs succeed or get skipped. - This will be the job that will be set on the branch protection policy + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + description: "JSON array of job IDs to run change detection for. Default 'all' runs for all jobs" +``` + +### 2. Set Up Environment Variables + +Add this environment configuration at the workflow level: -3. Add a `needs` statement that refers to the check-changes job (ex: `workdir-has-changes`) +```yaml +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} # Base directory for change detection + # Add your other environment variables here +``` -4. Add an `if` statement that refers to the changes output variable (ex: `needs.workdir-has-changes.outputs.run-backend`) and skips the job if it evaluates to false +### 3. Add the Change Detection Job -### Full example +Add this job to check for directory changes: -```yml +```yaml jobs: workdir-has-changes: runs-on: ubuntu-latest outputs: - run-backend: ${{ steps.filter.outputs.backend }} + changes-detected: ${{ steps.filter.outputs.changes-detected }} steps: + - name: Set CHECK_DIR to custom directory if provided + if: ${{ inputs.CHECK_CUSTOM_DIR != '' }} + run: echo "CHECK_DIR=${{ inputs.CHECK_CUSTOM_DIR }}" >> $GITHUB_ENV + - name: Set default CHECK_DIR + if: ${{ inputs.CHECK_CUSTOM_DIR == '' }} + run: echo "CHECK_DIR=${{ inputs.WORKING_DIRECTORY }}" >> $GITHUB_ENV + - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: | - backend: - - 'Backend/**' + changes-detected: + - "${{ env.CHECK_DIR }}/**" +``` - pr-backend: +### 4. Update Your Main Jobs + +Modify your workflow jobs to use change detection: + +```yaml +jobs: + your-job-name: needs: workdir-has-changes - if: ${{ needs.workdir-has-changes.outputs.run-backend == 'true' }} - uses: ./.github/workflows/common-backend.yml - secrets: inherit + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + runs-on: ubuntu-latest + steps: + # Your job steps here +``` +### 5. Add the Success Validation Job + +Add this job to validate the overall workflow status: + +```yaml +jobs: jobs-succeded: - needs: ["pr-backend"] + needs: ["your-job-name"] # List all jobs that need validation runs-on: ubuntu-latest - if: ${{ always()}} + if: ${{ always() }} steps: - - name: "Jobs: pr-backend didn't fail." - run: if [[ "${{ needs.pr-backend.result }}" == "failure" ]]; then exit 1; fi + - name: "Check if jobs succeeded" + run: if [[ "${{ needs.your-job-name.result }}" == "failure" ]]; then exit 1; fi ``` + +## Usage Examples + +### Basic Usage + +```yaml +name: Example Workflow +on: + workflow_call: + inputs: + WORKING_DIRECTORY: + required: true + type: string + CHECK_WORKDIR_CHANGES: + required: true + type: boolean + default: false + CHECK_CUSTOM_DIR: + required: false + type: string + default: "" + CHECK_CHANGES_BY_JOBS: + required: false + type: string + default: "all" + +env: + CHECK_DIR: ${{ inputs.WORKING_DIRECTORY }} + +jobs: + workdir-has-changes: + # Change detection job configuration as above + + main-job: + needs: workdir-has-changes + if: ${{ !inputs.CHECK_WORKDIR_CHANGES || (needs.workdir-has-changes.outputs.changes-detected == 'true' && (inputs.CHECK_CHANGES_BY_JOBS == 'all' || contains(fromJson(inputs.CHECK_CHANGES_BY_JOBS), github.job)))}} + runs-on: ubuntu-latest + steps: + - run: echo "Main job running" + + jobs-succeded: + needs: ["main-job"] + # Success validation job configuration as above +``` + +### Advanced Usage + +1. To run only specific jobs when changes are detected: + +```yaml +with: + CHECK_WORKDIR_CHANGES: true + CHECK_CHANGES_BY_JOBS: "['build', 'test']" +``` + +2. To check a different directory than the working directory: + +```yaml +with: + CHECK_WORKDIR_CHANGES: true + CHECK_CUSTOM_DIR: "src/frontend" +``` + +3. To disable change detection and run all jobs: + +```yaml +with: + CHECK_WORKDIR_CHANGES: false +``` + +## Important Notes + +1. The `CHECK_WORKDIR_CHANGES` parameter acts as a master switch: + + - When `false`: All jobs run regardless of changes + - When `true`: Jobs only run if changes are detected + +2. The `CHECK_CUSTOM_DIR` parameter allows checking a different directory than `WORKING_DIRECTORY` + +3. The `CHECK_CHANGES_BY_JOBS` parameter controls which jobs use change detection: + + - Default `"all"`: All jobs use change detection + - JSON array of job IDs: Only listed jobs use change detection + +4. The success validation job (`jobs-succeded`) should list all workflow jobs in its `needs` array + +5. Branch protection rules should reference the `jobs-succeded` job, not individual workflow jobs + +6. The `CHECK_DIR` environment variable is used as the base directory for change detection and can be overridden by `CHECK_CUSTOM_DIR`