CodeQL #12
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CodeQL" | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| # The branches below must be a subset of the branches above | |
| branches: [ master ] | |
| schedule: | |
| - cron: '32 10 * * 5' | |
| concurrency: | |
| # This controls which concurrent builds to cancel: | |
| # - we do not want any concurrent builds on a branch (pull_request) | |
| # - we do not want concurrent builds on the same commit on master (push) | |
| # - we do not want concurrent builds on the same commit on a tag (push) | |
| # - we allow concurrent runs on the same commit on master and its tag (push) | |
| # - we allow concurrent runs on the same commit on master (push) and a scheduled build (schedule) | |
| # | |
| # A pull_request event only runs on branch commit, a push event only on master and tag commit. | |
| # A schedule event only runs on master HEAD commit. | |
| # | |
| # Expression github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch. | |
| # This helps to not cancel concurrent runs on master or a tag that share the same commit. | |
| # Expression github.head_ref refers to the branch of the pull request. | |
| # On master, github.head_ref is empty, so we use the SHA of the commit, this means individual | |
| # commits to master will not be cancelled, while there can only be one concurrent build on a branch. | |
| # | |
| # We include the event name to we allow for concurrent scheduled and master builds. | |
| group: codeql-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| analyze-python: | |
| name: Analyze Python | |
| runs-on: ubuntu-latest | |
| permissions: | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Install Python dependencies | |
| # install all `install_requires` dependencies but no `extras_require` dependencies | |
| run: | | |
| python setup.py egg_info | |
| pip install $(head -n $(( $(grep -m 1 -n -e "^\[" horovod.egg-info/requires.txt | cut -d : -f 1) - 1 )) horovod.egg-info/requires.txt) | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v2 | |
| with: | |
| languages: python | |
| setup-python-dependencies: false | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v2 | |
| analyze-cpp: | |
| name: Analyze C++ | |
| runs-on: ubuntu-latest | |
| permissions: | |
| security-events: write | |
| steps: | |
| - name: Clean up disk space | |
| # deleting these paths frees 38 GB disk space: | |
| # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| # but this sometimes takes 3-4 minutes | |
| # so we delete only some sub-paths which are known to be quick (10s) and 20 GB | |
| run: | | |
| echo ::group::Disk space before clean up | |
| df -h | |
| echo ::endgroup:: | |
| for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \ | |
| /usr/share/dotnet/shared \ | |
| /usr/local/lib/android/sdk/ndk \ | |
| /usr/local/lib/android/sdk/build-tools \ | |
| /opt/ghc | |
| do | |
| echo ::group::Deleting "$dir" | |
| sudo du -hsc $dir | tail -n1 || true | |
| sudo rm -rf $dir | |
| echo ::endgroup:: | |
| done | |
| echo ::group::Disk space after clean up | |
| df -h | |
| echo ::endgroup:: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v2 | |
| with: | |
| languages: cpp | |
| - name: Add CodeQL to Dockerfile.test.?pu | |
| run: | | |
| # copy CodeQL distribution into docker context | |
| cp -rl "${{ env.CODEQL_DIST }}" .codeql | |
| command=$(cat <<EOF | |
| # Install CodeQL | |
| COPY .codeql ${{ env.CODEQL_DIST }} | |
| RUN ${{ env.CODEQL_DIST }}/codeql version --format=terse | |
| RUN ${{ env.CODEQL_DIST }}/codeql version --format=json | |
| RUN ${{ env.CODEQL_DIST }}/codeql resolve queries cpp-code-scanning.qls --format=bylanguage | |
| RUN ${{ env.CODEQL_DIST }}/codeql resolve languages | |
| RUN ${{ env.CODEQL_DIST }}/codeql resolve qlpacks | |
| EOF | |
| ) | |
| sed -i -e "s%^# setup ssh service$%${command//$'\n'/\\n}\n\n# setup ssh service%" Dockerfile.test.?pu | |
| command=$(cat <<EOF | |
| # Setup CodeQL tracing | |
| RUN mkdir -p /home/runner/work/horovod | |
| RUN ln -s /horovod /home/runner/work/horovod/horovod | |
| RUN mkdir -p /home/runner/work/_temp | |
| RUN ${{ env.CODEQL_DIST }}/codeql database init --db-cluster /home/runner/work/_temp/codeql_databases --source-root=/home/runner/work/horovod/horovod --language=cpp --begin-tracing --trace-process-name=Runner.Worker.exe | |
| # Build Horovod (C++) | |
| RUN mkdir -p build/temp | |
| RUN cd build/temp; source /home/runner/work/_temp/codeql_databases/temp/tracingEnvironment/start-tracing.sh; cmake /home/runner/work/horovod/horovod -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO=/home/runner/work/horovod/horovod/build/temp/lib.linux-x86_64-3.8 -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python | |
| RUN cd build/temp; source /home/runner/work/_temp/codeql_databases/temp/tracingEnvironment/start-tracing.sh; cmake --build . --config RelWithDebInfo -- VERBOSE=1 | |
| EOF | |
| ) | |
| sed -i -e "s%^# Install Horovod.$%${command//$'\n'/\\n}\n\n# Install Horovod.%" Dockerfile.test.?pu | |
| # Truncate Dockerfile.test.?pu, after CMake we are done here | |
| for file in Dockerfile.test.?pu | |
| do | |
| head -n $(( $(grep -m 1 -n "# Install Horovod." $file | cut -d : -f 1) - 1 )) $file > tmp | |
| mv tmp $file | |
| done | |
| # Print out changes | |
| git diff | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| - name: Build Horovod | |
| shell: bash | |
| run: | | |
| pip install docker-compose | |
| image=$(grep mixed docker-compose.test.yml | sed -e "s/[ :]//g") | |
| docker-compose -f docker-compose.test.yml build ${image} | |
| docker run --name horovod horovod_${image} ls -lah /home/runner/work/_temp/codeql_databases | |
| rm -rf /home/runner/work/_temp/codeql_databases | |
| docker cp horovod:/home/runner/work/_temp/codeql_databases /home/runner/work/_temp/ | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v2 |