From d0091bb4f20f9d1cdc051c104187aad240bce2ea Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 28 May 2024 16:17:59 -0700 Subject: [PATCH 01/12] Add first-pass CI to run womtool on WDL files --- .github/workflows/womtool.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/womtool.yml diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml new file mode 100644 index 0000000..451489b --- /dev/null +++ b/.github/workflows/womtool.yml @@ -0,0 +1,39 @@ +name: Womtool + +# Enable Buildkit and let compose use it to speed up image building +env: + DOCKER_BUILDKIT: 1 + COMPOSE_DOCKER_CLI_BUILD: 1 + +on: + pull_request: + branches: [ "main" ] + paths-ignore: [ "docs/**" ] + + push: + branches: [ "main" ] + paths-ignore: [ "docs/**" ] + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + + run-womtool: + runs-on: ubuntu-latest + strategy: + matrix: + wdl-file: ["validate_gsr_model.wdl"] + + name: "Run womtool: ${{ matrix.wdl_file }}" + + steps: + + - name: Checkout Code Repository + uses: actions/checkout@v4 + + - name: Run womtool + uses: UW-GAC/womtool-action@main + with: + wdl-file: ${{ matrix.wdl_file }} From 8e4430aa8eb1216552372a2aabd7641bf0fa94e2 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 28 May 2024 16:22:52 -0700 Subject: [PATCH 02/12] Fix matrix context accessing - instead of _ --- .github/workflows/womtool.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 451489b..048b931 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -26,7 +26,7 @@ jobs: matrix: wdl-file: ["validate_gsr_model.wdl"] - name: "Run womtool: ${{ matrix.wdl_file }}" + name: "Run womtool: ${{ matrix.wdl-file }}" steps: @@ -36,4 +36,4 @@ jobs: - name: Run womtool uses: UW-GAC/womtool-action@main with: - wdl-file: ${{ matrix.wdl_file }} + wdl-file: ${{ matrix.wdl-file }} From 092f74032ecc4f1585a2f5a0c4175eb27b4f9679 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 28 May 2024 16:28:06 -0700 Subject: [PATCH 03/12] Intentionally break gsr data model to test womtool CI --- validate_gsr_model.wdl | 1 - 1 file changed, 1 deletion(-) diff --git a/validate_gsr_model.wdl b/validate_gsr_model.wdl index 89fbe80..9f432a4 100644 --- a/validate_gsr_model.wdl +++ b/validate_gsr_model.wdl @@ -67,7 +67,6 @@ workflow validate_gsr_model { task validate { input { - Map[String, File] table_files String model_url String workspace_name String workspace_namespace From ae0e603629cbe14545138364844926242966a874 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Tue, 28 May 2024 16:29:42 -0700 Subject: [PATCH 04/12] Undo intentional breaking change to gsr data model --- validate_gsr_model.wdl | 1 + 1 file changed, 1 insertion(+) diff --git a/validate_gsr_model.wdl b/validate_gsr_model.wdl index 9f432a4..89fbe80 100644 --- a/validate_gsr_model.wdl +++ b/validate_gsr_model.wdl @@ -67,6 +67,7 @@ workflow validate_gsr_model { task validate { input { + Map[String, File] table_files String model_url String workspace_name String workspace_namespace From 018981221a6ea79a90e1b154f571a8efb93316d8 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 09:54:25 -0700 Subject: [PATCH 05/12] Add all WDL files to CI --- .github/workflows/womtool.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 048b931..0cc2420 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -24,7 +24,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - wdl-file: ["validate_gsr_model.wdl"] + wdl-file: [ + ./gsr_data_report.wdl, + ./check_vcf_samples.wdl, + ./pheno_qc/pheno_qc.wdl, + ./validate_genotype_model.wdl + ./validate_gsr_model.wdl, + ./validate_phenotype_model.wdl, + ] name: "Run womtool: ${{ matrix.wdl-file }}" From 2d5097ecc4b988a94e4ecd8f7117e4847a2fc8ab Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 09:55:45 -0700 Subject: [PATCH 06/12] Run all womtool jobs regardless of whether one WDL fails --- .github/workflows/womtool.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 0cc2420..696ec85 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -23,6 +23,7 @@ jobs: run-womtool: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: wdl-file: [ ./gsr_data_report.wdl, From 747ed300a14a223fd43aff6b500f616b797834f0 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 09:56:44 -0700 Subject: [PATCH 07/12] Add missing comma --- .github/workflows/womtool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 696ec85..e0b2f4e 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -29,7 +29,7 @@ jobs: ./gsr_data_report.wdl, ./check_vcf_samples.wdl, ./pheno_qc/pheno_qc.wdl, - ./validate_genotype_model.wdl + ./validate_genotype_model.wdl, ./validate_gsr_model.wdl, ./validate_phenotype_model.wdl, ] From ac5de0a502b89fc9da2bb11545153ac17754167a Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 10:04:46 -0700 Subject: [PATCH 08/12] Try detecting WDL files automatically in CI --- .github/workflows/womtool.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index e0b2f4e..9ee2238 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -20,19 +20,31 @@ concurrency: jobs: + detect-wdl-files: + runs-on: ubuntu-latest + name: "Detect WDL files" + outputs: + matrix: ${{ steps.detect-wdl-files.outputs.wdl-file }} + + steps: + + - name: Checkout Code Repository + uses: actions/checkout@v4 + + - name: Detect WDL files + id: detect-wdl-files + run: | + echo "::set-output name=wdl-file::$(find . -name '*.wdl' -type f -print0 | xargs -0)" + + - name: Show WDL files + run: echo "${{ steps.detect-wdl-files.outputs.wdl-file }}" + run-womtool: + needs: detect-wdl-files runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - wdl-file: [ - ./gsr_data_report.wdl, - ./check_vcf_samples.wdl, - ./pheno_qc/pheno_qc.wdl, - ./validate_genotype_model.wdl, - ./validate_gsr_model.wdl, - ./validate_phenotype_model.wdl, - ] + matrix: ${{ fromJSON(needs.detect-wdl-files.outputs.matrix) }} name: "Run womtool: ${{ matrix.wdl-file }}" From 6bdfe441e4893c597cd71c7888a70ecd2629a5a8 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 10:13:58 -0700 Subject: [PATCH 09/12] Try new way of specifying matrix values in detect-wdl-files job --- .github/workflows/womtool.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 9ee2238..9b398ac 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -34,7 +34,12 @@ jobs: - name: Detect WDL files id: detect-wdl-files run: | - echo "::set-output name=wdl-file::$(find . -name '*.wdl' -type f -print0 | xargs -0)" + # Find the WDL files + wdl_files=`find . -name "*.wdl" -type f -print0 | xargs -0` + # Add double quotes and replace spaces with commas. + wdl_files=\"${wdl_files// /\",\"}\" + + echo "matrix={\"wdl-file\":[$wdl_files]" >> $GITHUB_OUTPUT - name: Show WDL files run: echo "${{ steps.detect-wdl-files.outputs.wdl-file }}" From f4ed263614e9f477d006cc681d0de5627cd98864 Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 10:14:54 -0700 Subject: [PATCH 10/12] Show the correct output --- .github/workflows/womtool.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 9b398ac..870454b 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -42,7 +42,7 @@ jobs: echo "matrix={\"wdl-file\":[$wdl_files]" >> $GITHUB_OUTPUT - name: Show WDL files - run: echo "${{ steps.detect-wdl-files.outputs.wdl-file }}" + run: echo "${{ steps.detect-wdl-files.outputs.matrix }}" run-womtool: needs: detect-wdl-files From db1c449d6bb181f957c4107bc09426622bd89fdb Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 10:41:14 -0700 Subject: [PATCH 11/12] Fix output value for detect-wdl-files job --- .github/workflows/womtool.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 870454b..02acf2c 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest name: "Detect WDL files" outputs: - matrix: ${{ steps.detect-wdl-files.outputs.wdl-file }} + matrix: ${{ steps.detect-files.outputs.matrix }} steps: @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Detect WDL files - id: detect-wdl-files + id: detect-files run: | # Find the WDL files wdl_files=`find . -name "*.wdl" -type f -print0 | xargs -0` @@ -42,7 +42,7 @@ jobs: echo "matrix={\"wdl-file\":[$wdl_files]" >> $GITHUB_OUTPUT - name: Show WDL files - run: echo "${{ steps.detect-wdl-files.outputs.matrix }}" + run: echo "${{ steps.detect-files.outputs.matrix }}" run-womtool: needs: detect-wdl-files From c26567113e0efe42acdb6f50d16c64a223c2e5ba Mon Sep 17 00:00:00 2001 From: Adrienne Stilp Date: Wed, 29 May 2024 10:44:08 -0700 Subject: [PATCH 12/12] Go back to hardcoding WDL files in CI I couldn't get the dynamic detection to work, so go back to the simple method. --- .github/workflows/womtool.yml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/womtool.yml b/.github/workflows/womtool.yml index 02acf2c..e0b2f4e 100644 --- a/.github/workflows/womtool.yml +++ b/.github/workflows/womtool.yml @@ -20,36 +20,19 @@ concurrency: jobs: - detect-wdl-files: - runs-on: ubuntu-latest - name: "Detect WDL files" - outputs: - matrix: ${{ steps.detect-files.outputs.matrix }} - - steps: - - - name: Checkout Code Repository - uses: actions/checkout@v4 - - - name: Detect WDL files - id: detect-files - run: | - # Find the WDL files - wdl_files=`find . -name "*.wdl" -type f -print0 | xargs -0` - # Add double quotes and replace spaces with commas. - wdl_files=\"${wdl_files// /\",\"}\" - - echo "matrix={\"wdl-file\":[$wdl_files]" >> $GITHUB_OUTPUT - - - name: Show WDL files - run: echo "${{ steps.detect-files.outputs.matrix }}" - run-womtool: - needs: detect-wdl-files runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJSON(needs.detect-wdl-files.outputs.matrix) }} + matrix: + wdl-file: [ + ./gsr_data_report.wdl, + ./check_vcf_samples.wdl, + ./pheno_qc/pheno_qc.wdl, + ./validate_genotype_model.wdl, + ./validate_gsr_model.wdl, + ./validate_phenotype_model.wdl, + ] name: "Run womtool: ${{ matrix.wdl-file }}"