C#/Java: Content based model generation improvements. #515
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Models as Data - Diff | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| projects: | |
| description: "The projects to generate models for" | |
| required: true | |
| default: '["netty/netty"]' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - "java/ql/src/utils/modelgenerator/**/*.*" | |
| - "misc/scripts/models-as-data/*.*" | |
| - ".github/workflows/mad_modelDiff.yml" | |
| permissions: | |
| contents: read | |
| jobs: | |
| model-diff: | |
| name: Model Difference | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'github/codeql' | |
| strategy: | |
| matrix: | |
| slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}} | |
| steps: | |
| - name: Clone github/codeql from PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request | |
| with: | |
| path: codeql-pr | |
| - name: Clone github/codeql from main | |
| uses: actions/checkout@v4 | |
| with: | |
| path: codeql-main | |
| ref: main | |
| - uses: ./codeql-main/.github/actions/fetch-codeql | |
| - name: Download database | |
| env: | |
| SLUG: ${{ matrix.slug }} | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -x | |
| mkdir lib-dbs | |
| SHORTNAME=${SLUG//[^a-zA-Z0-9_]/} | |
| gh api -H "Accept: application/zip" "/repos/${SLUG}/code-scanning/codeql/databases/java" > "$SHORTNAME.zip" | |
| unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip" | |
| mkdir "lib-dbs/$SHORTNAME/" | |
| mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/" | |
| - name: Generate Models (PR and main) | |
| run: | | |
| set -x | |
| mkdir tmp-models | |
| MODELS=`pwd`/tmp-models | |
| DATABASES=`pwd`/lib-dbs | |
| analyzeDatabaseWithCheckout() { | |
| QL_VARIANT=$1 | |
| DATABASE=$2 | |
| cd codeql-$QL_VARIANT | |
| SHORTNAME=`basename $DATABASE` | |
| python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT | |
| mkdir -p $MODELS/$SHORTNAME | |
| mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME | |
| cd .. | |
| } | |
| for d in $DATABASES/*/ ; do | |
| ls -1 "$d" | |
| analyzeDatabaseWithCheckout "main" $d | |
| if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] | |
| then | |
| analyzeDatabaseWithCheckout "pr" $d | |
| fi | |
| done | |
| - name: Install diff2html | |
| if: github.event.pull_request | |
| run: | | |
| npm install -g diff2html-cli | |
| - name: Generate Model Diff | |
| if: github.event.pull_request | |
| run: | | |
| set -x | |
| MODELS=`pwd`/tmp-models | |
| ls -1 tmp-models/ | |
| for m in $MODELS/*/main/*.model.yml ; do | |
| t="${m/main/"pr"}" | |
| basename=`basename $m` | |
| name="diff_${basename/.model.yml/""}" | |
| (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true | |
| done | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: models | |
| path: tmp-models/**/**/*.model.yml | |
| retention-days: 20 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: diffs | |
| path: tmp-models/*.html | |
| # An html file is only produced if the generated models differ. | |
| if-no-files-found: ignore | |
| retention-days: 20 |