This repository has been archived by the owner on Aug 25, 2024. It is now read-only.
chore(deps): bump ubuntu from 20.04 to 22.04 in /dffml/skel/operations #5047
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| workflow_dispatch: null | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.12" | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dev dependencies | |
| run: | | |
| pip install -U pip setuptools wheel build | |
| pip install -U sbom4python | |
| pip install -U https://github.com/scitt-community/scitt-api-emulator/archive/e89a60584fa717382f279ae24b8a1a93d458bb4d.zip | |
| pip install -e .[dev] | |
| python -m pip freeze | |
| - name: Build | |
| run: | | |
| python -m build . | |
| - name: Generate SBOM | |
| id: generate-sbom | |
| uses: anthonyharrison/sbom4python@5b458354df89357bf0253e62ea4567b1807120e2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| module-name: dffml | |
| output-directory: sbom | |
| - name: in-toto attestation for cyclonedx SBOM | |
| id: in-toto-cyclonedx | |
| env: | |
| MODULE_NAME: dffml | |
| run: | | |
| echo "attestation<<GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT | |
| (python -m json.tool --sort-keys | tee -a $GITHUB_OUTPUT) <<EOF | |
| { | |
| "_type": "https://in-toto.io/Statement/v0.1", | |
| "subject": [ | |
| { | |
| "name": "$(cd dist/ && echo *.tar.gz)", | |
| "digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.tar.gz) | awk '{print $1}')"} | |
| }, | |
| { | |
| "name": "$(cd dist/ && echo *.whl)", | |
| "digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.whl) | awk '{print $1}')"} | |
| } | |
| ], | |
| "predicateType": "https://cyclonedx.org/bom/v1.4", | |
| "predicate": $(cat "${MODULE_NAME}-py${{ matrix.python-version }}.json") | |
| } | |
| EOF | |
| echo "GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT | |
| - name: Checkout public-keys branch | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| ref: public-keys | |
| path: public-keys | |
| - name: Generate keypair to sign SCITT statement | |
| id: scitt-gen-keypair | |
| run: | | |
| ssh-keygen -q -f ssh-private -t ecdsa -b 384 -N '' -C "$(head -n 100 /dev/urandom | sha384sum | awk '{print $1}')" -I "$(date -Iseconds)" <<<y | |
| cat ssh-private | python -c 'import sys; from cryptography.hazmat.primitives import serialization; print(serialization.load_ssh_private_key(sys.stdin.buffer.read(), password=None).private_bytes(encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()).decode().rstrip())' > private-key.pem | |
| # cat ssh-private.pub | tee -a public-keys/x509 | |
| - name: SSH key as x509 | |
| shell: python -u {0} | |
| run: | | |
| from cryptography import x509 | |
| from cryptography.x509.oid import NameOID | |
| from cryptography.hazmat.primitives import hashes | |
| from cryptography.hazmat.backends import default_backend | |
| from cryptography.hazmat.primitives.asymmetric import rsa | |
| from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat | |
| from jwcrypto import jwk, jws | |
| import datetime | |
| import pathlib | |
| key = jwk.JWK.from_pem(pathlib.Path("private-key.pem").read_bytes()) | |
| rsa_public_key = jwk.JWK.from_json(key.export_public()) | |
| # Convert the JWK to a public key | |
| public_key = rsa_public_key.get_op_key('verify') | |
| # Create a builder for the X.509 certificate | |
| subject = issuer = x509.Name([ | |
| x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), | |
| x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Oregon"), | |
| x509.NameAttribute(NameOID.LOCALITY_NAME, "Portland"), | |
| x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SCITT Emulator"), | |
| x509.NameAttribute(NameOID.COMMON_NAME, "example.com"), | |
| ]) | |
| cert_builder = x509.CertificateBuilder( | |
| subject_name=subject, | |
| issuer_name=issuer, | |
| public_key=public_key, | |
| serial_number=x509.random_serial_number(), | |
| not_valid_before=datetime.datetime.utcnow(), | |
| not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=1), # Certificate valid for 1 day | |
| extensions=[] | |
| ) | |
| # Self-sign the certificate with the private key | |
| private_key_op = key.get_op_key('sign') | |
| cert = cert_builder.sign(private_key=private_key_op, algorithm=hashes.SHA256(), backend=default_backend()) | |
| # Serialize the certificate | |
| cert_pem = cert.public_bytes(encoding=Encoding.PEM) | |
| # Save the PEM encoded certificate | |
| pathlib.Path("public-keys", "x509").write_bytes(cert_pem) | |
| - name: Remove ssh private | |
| run: | | |
| rm -v ssh-private | |
| - name: Push new public key | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -xe | |
| cd public-keys | |
| gh auth setup-git | |
| git config --global --add safe.directory "${PWD}" | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "GitHub Actions" | |
| git add -A | |
| # If no delta clean exit | |
| git commit -sm "Snapshot" || exit 0 | |
| git push -uf origin "HEAD:public-keys" | |
| # Wait for propagation | |
| # TODO x509 style polling? | |
| sleep 10s | |
| - name: Submit SBOM to SCITT | |
| id: scitt-submit-sbom | |
| uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654 | |
| with: | |
| issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509 | |
| subject: pkg:github/${{ github.repository }}@${{ github.sha }} | |
| payload: ${{ steps.in-toto-cyclonedx.outputs.attestation }} | |
| private-key-pem: private-key.pem | |
| scitt-url: https://scitt.unstable.chadig.com | |
| - name: Create Pull Request | |
| if: ${{ steps.generate-sbom.outputs.changed }} | |
| uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 | |
| with: | |
| commit-message: "chore: update SBOM for Python ${{ matrix.python-version }}" | |
| title: "chore: update SBOM for Python ${{ matrix.python-version }}" | |
| branch: chore-sbom-py${{ matrix.python-version }} | |
| delete-branch: true | |
| author: GitHub Actions <[email protected]> | |
| add-paths: sbom | |
| - uses: openvex/generate-vex@159b7ee4845fb48f1991395ce8501d6263407360 | |
| name: Run vexctl | |
| id: vexctl | |
| with: | |
| product: pkg:github/${{ github.repository }}@${{ github.sha }} | |
| - name: Submit OpenVEX to SCITT | |
| id: scitt-submit-openvex | |
| uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654 | |
| with: | |
| issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509 | |
| subject: pkg:github/${{ github.repository }}@${{ github.sha }} | |
| payload: ${{ steps.vexctl.outputs.openvex }} | |
| private-key-pem: private-key.pem | |
| scitt-url: https://scitt.unstable.chadig.com | |
| - name: Remove private key used in keypair to sign SCITT statement | |
| run: | | |
| rm -v private-key.pem | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 40 | |
| matrix: | |
| check: [changelog, whitespace, commit, lines] | |
| python-version: [3.7] | |
| node-version: [12.x] | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.check }} | |
| - name: Run check | |
| run: | | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }} | |
| container: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - | |
| sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | |
| sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io | |
| - name: Run check | |
| run: | | |
| ./.ci/run.sh container | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| plugin: | |
| - model/daal4py | |
| - model/tensorflow | |
| - model/scratch | |
| - model/scikit | |
| - model/vowpalWabbit | |
| - model/autosklearn | |
| - model/xgboost | |
| - operations/binsec | |
| - operations/data | |
| - operations/deploy | |
| - operations/image | |
| - operations/nlp | |
| - source/mysql | |
| - feature/git | |
| - feature/auth | |
| - service/http | |
| - configloader/yaml | |
| - configloader/image | |
| - source/mongodb | |
| - entities/alice | |
| python-version: | |
| - "3.12" | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Remove unused software | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: shouldi test binary cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: examples/shouldi/tests/downloads | |
| key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-shouldi-test-binaries- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.plugin }} | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| export PYPI_TOKENS=$(mktemp) | |
| cat > ${PYPI_TOKENS} <<EOF | |
| examples/shouldi=${{ secrets.PYPI_SHOULDI }} | |
| model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }} | |
| model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }} | |
| model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }} | |
| model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }} | |
| model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }} | |
| model/spacy=${{ secrets.PYPI_MODEL_SPACY }} | |
| model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }} | |
| model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }} | |
| model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }} | |
| model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }} | |
| source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }} | |
| source/mongodb=${{ secrets.PYPI_SOURCE_MONGODB }} | |
| feature/git=${{ secrets.PYPI_FEATURE_GIT }} | |
| feature/auth=${{ secrets.PYPI_FEATURE_AUTH }} | |
| operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }} | |
| operations/data=${{ secrets.PYPI_OPERATIONS_DATA }} | |
| operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }} | |
| operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }} | |
| operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }} | |
| service/http=${{ secrets.PYPI_SERVICE_HTTP }} | |
| configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }} | |
| configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }} | |
| EOF | |
| export TWINE_USERNAME=__token__ | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }} | |
| else | |
| export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" | sed 's/^[^=]*=//g') | |
| fi | |
| rm ${PYPI_TOKENS} | |
| ./.ci/run.sh "${PLUGIN}" | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov | |
| fi | |
| tutorials: | |
| runs-on: ubuntu-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| python-version: [3.7] | |
| docs: | |
| - docs/cli.rst | |
| - docs/examples/dataflows.rst | |
| - docs/examples/integration.rst | |
| - docs/examples/or_covid_data_by_county.rst | |
| - docs/examples/shouldi.rst | |
| - docs/examples/innersource/swportal.rst | |
| - docs/examples/innersource/microservice.rst | |
| - docs/examples/innersource/kubernetes.rst | |
| - docs/examples/icecream_sales.rst | |
| - docs/examples/data_cleanup/data_cleanup.rst | |
| - docs/examples/data_cleanup/data_cleanup_classfication.rst | |
| - docs/installation.rst | |
| - docs/troubleshooting.rst | |
| - docs/tutorials/accuracy/mse.rst | |
| - docs/tutorials/dataflows/io.rst | |
| - docs/tutorials/dataflows/nlp.rst | |
| - docs/tutorials/models/iris.rst | |
| - docs/tutorials/models/package.rst | |
| - docs/tutorials/models/docs.rst | |
| - docs/tutorials/models/slr.rst | |
| - docs/tutorials/sources/complex.rst | |
| - docs/tutorials/sources/file.rst | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh . | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| ./.ci/run.sh consoletest ${{ matrix.docs }} | |
| macos: | |
| runs-on: macos-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.7] | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Setup DFFML | |
| run: | | |
| pip install -U pip setuptools wheel | |
| pip install -e .[dev] | |
| dffml service dev install -skip model/daal4py | |
| # XGBoost requires libomp on OSX | |
| brew install libomp | |
| - name: Test | |
| run: | | |
| python -m unittest discover -v | |
| python -m pip freeze |