Skip to content

HJ-250 - Address PyPi deprecation warnings #12027

HJ-250 - Address PyPi deprecation warnings

HJ-250 - Address PyPi deprecation warnings #12027

Workflow file for this run

name: Backend Code Checks
on:
pull_request:
# This only ignores paths if there are _no_ commits on the branch that touch other files
paths-ignore:
- "**.md"
- "clients/**"
- ".vscode/**"
- ".github/workflows/frontend_checks.yml"
push:
branches:
- "main"
- "release-**"
env:
IMAGE: ethyca/fides:local
DEFAULT_PYTHON_VERSION: "3.10.13"
# Docker auth with read-only permissions.
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }}
jobs:
###############
## Prechecks ##
###############
Collect-Tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Run Static Check
run: nox -s collect_tests
Build:
needs: Collect-Tests
strategy:
matrix:
# NOTE: These are the currently supported/tested Python Versions
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build container
uses: docker/build-push-action@v5
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
build-args: PYTHON_VERSION=${{ matrix.python_version }}
target: prod
outputs: type=docker,dest=/tmp/python-${{ matrix.python_version }}.tar
push: false
tags: ${{ env.IMAGE }}
- name: Upload container
uses: actions/upload-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/python-${{ matrix.python_version }}.tar
retention-days: 1
###################
## Static Checks ##
###################
Static-Checks:
strategy:
matrix:
session_name:
[
'"isort(check)"',
'"black(check)"',
"mypy",
"pylint",
"xenon",
"check_install",
'"pytest(nox)"',
]
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Install Dev Requirements
run: pip install -r dev-requirements.txt
- name: Run Static Check
run: nox -s ${{ matrix.session_name }}
##################
## Performance ##
##################
Performance-Checks:
needs: Check-Container-Startup
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Rust/Cargo
run: curl -y https://sh.rustup.rs -sSf | sh
- name: Install Drill
run: cargo install drill
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Run Performance Tests
run: nox -s performance_tests
#################
## Misc Checks ##
#################
Check-Container-Startup:
needs: Build
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Run test suite
run: nox -s check_container_startup
Misc-Tests:
needs: Check-Container-Startup
strategy:
matrix:
test_selection:
- "check_fides_annotations"
- "fides_db_scan"
- "docs_check"
- "minimal_config_startup"
runs-on: ubuntu-latest
timeout-minutes: 15
continue-on-error: true
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Run test suite
run: nox -s "${{ matrix.test_selection }}"
################
## Safe Tests ##
################
Safe-Tests:
needs: Check-Container-Startup
strategy:
fail-fast: false
matrix:
python_version: ["3.9.18", "3.10.13"]
test_selection:
- "ctl-not-external"
- "ops-unit-api"
- "ops-unit-non-api"
- "ops-integration"
- "lib"
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Run test suite
run: nox -s "pytest(${{ matrix.test_selection }})"
- name: Upload coverage
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
##################
## Unsafe Tests ##
##################
# NOTE: Matrixes aren't used here due to the danger of race conditions for external resources
Pytest-Ctl-External:
needs: Check-Container-Startup
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
timeout-minutes: 20
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Load secrets from 1Password
uses: 1password/load-secrets-action@v2
with:
# Export loaded secrets as environment variables
export-env: true
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
# Secrets to pull from 1Password
AWS_ACCESS_KEY_ID: op://github-actions/ctl/AWS_ACCESS_KEY_ID
AWS_DEFAULT_REGION: op://github-actions/ctl/AWS_DEFAULT_REGION
AWS_SECRET_ACCESS_KEY: op://github-actions/ctl/AWS_SECRET_ACCESS_KEY
DYNAMODB_ACCESS_KEY_ID: op://github-actions/dynamodb/DYNAMODB_ACCESS_KEY_ID
DYNAMODB_ACCESS_KEY: op://github-actions/dynamodb/DYNAMODB_ACCESS_KEY
DYNAMODB_REGION: op://github-actions/dynamodb/DYNAMODB_REGION
OKTA_CLIENT_TOKEN: op://github-actions/ctl/OKTA_CLIENT_TOKEN
REDSHIFT_FIDESCTL_PASSWORD: op://github-actions/ctl/REDSHIFT_FIDESCTL_PASSWORD
SNOWFLAKE_FIDESCTL_PASSWORD: op://github-actions/ctl/SNOWFLAKE_FIDESCTL_PASSWORD
- name: Run external test suite
run: nox -s "pytest(ctl-external)"
env:
BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }}
External-Datastores:
needs: Check-Container-Startup
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
timeout-minutes: 20
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Install Nox
run: pip install nox>=2022
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: Load secrets from 1Password
uses: 1password/load-secrets-action@v2
with:
# Export loaded secrets as environment variables
export-env: true
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
# Secrets to pull from 1Password
BIGQUERY_DATASET: op://github-actions/bigquery/BIGQUERY_DATASET
BIGQUERY_KEYFILE_CREDS: op://github-actions/bigquery/BIGQUERY_KEYFILE_CREDS
BIGQUERY_ENTERPRISE_DATASET: op://github-actions/bigquery-enterprise/BIGQUERY_ENTERPRISE_DATASET
BIGQUERY_ENTERPRISE_KEYFILE_CREDS: op://github-actions/bigquery-enterprise/BIGQUERY_ENTERPRISE_KEYFILE_CREDS
DYNAMODB_ACCESS_KEY_ID: op://github-actions/dynamodb/DYNAMODB_ACCESS_KEY_ID
DYNAMODB_ACCESS_KEY: op://github-actions/dynamodb/DYNAMODB_ACCESS_KEY
DYNAMODB_ASSUME_ROLE_ARN: op://github-actions/dynamodb/DYNAMODB_ASSUME_ROLE_ARN
DYNAMODB_REGION: op://github-actions/dynamodb/DYNAMODB_REGION
GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME: op://github-actions/gcp-mysql/GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME
GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER: op://github-actions/gcp-mysql/GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER
GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME: op://github-actions/gcp-mysql/GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME
GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS: op://github-actions/gcp-mysql/GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS
GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_NAME: op://github-actions/gcp-postgres/GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_NAME
GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_SCHEMA_NAME: op://github-actions/gcp-postgres/GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_SCHEMA_NAME
GOOGLE_CLOUD_SQL_POSTGRES_DB_IAM_USER: op://github-actions/gcp-postgres/GOOGLE_CLOUD_SQL_POSTGRES_DB_IAM_USER
GOOGLE_CLOUD_SQL_POSTGRES_INSTANCE_CONNECTION_NAME: op://github-actions/gcp-postgres/GOOGLE_CLOUD_SQL_POSTGRES_INSTANCE_CONNECTION_NAME
GOOGLE_CLOUD_SQL_POSTGRES_KEYFILE_CREDS: op://github-actions/gcp-postgres/GOOGLE_CLOUD_SQL_POSTGRES_KEYFILE_CREDS
RDS_MYSQL_AWS_ACCESS_KEY_ID: op://github-actions/rds-mysql/RDS_MYSQL_AWS_ACCESS_KEY_ID
RDS_MYSQL_AWS_SECRET_ACCESS_KEY: op://github-actions/rds-mysql/RDS_MYSQL_AWS_SECRET_ACCESS_KEY
RDS_MYSQL_DB_INSTANCE: op://github-actions/rds-mysql/RDS_MYSQL_DB_INSTANCE
RDS_MYSQL_DB_NAME: op://github-actions/rds-mysql/RDS_MYSQL_DB_NAME
RDS_MYSQL_DB_USERNAME: op://github-actions/rds-mysql/RDS_MYSQL_DB_USERNAME
RDS_MYSQL_REGION: op://github-actions/rds-mysql/RDS_MYSQL_REGION
RDS_POSTGRES_AWS_ACCESS_KEY_ID: op://github-actions/rds-postgres/RDS_POSTGRES_AWS_ACCESS_KEY_ID
RDS_POSTGRES_AWS_SECRET_ACCESS_KEY: op://github-actions/rds-postgres/RDS_POSTGRES_AWS_SECRET_ACCESS_KEY
RDS_POSTGRES_DB_USERNAME: op://github-actions/rds-postgres/RDS_POSTGRES_DB_USERNAME
RDS_POSTGRES_REGION: op://github-actions/rds-postgres/RDS_POSTGRES_REGION
REDSHIFT_TEST_DATABASE: op://github-actions/redshift/REDSHIFT_TEST_DATABASE
REDSHIFT_TEST_DB_SCHEMA: op://github-actions/redshift/REDSHIFT_TEST_DB_SCHEMA
REDSHIFT_TEST_HOST: op://github-actions/redshift/REDSHIFT_TEST_HOST
REDSHIFT_TEST_PASSWORD: op://github-actions/redshift/REDSHIFT_TEST_PASSWORD
REDSHIFT_TEST_PORT: op://github-actions/redshift/REDSHIFT_TEST_PORT
REDSHIFT_TEST_USER: op://github-actions/redshift/REDSHIFT_TEST_USER
SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER: op://github-actions/snowflake/SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER
SNOWFLAKE_TEST_DATABASE_NAME: op://github-actions/snowflake/SNOWFLAKE_TEST_DATABASE_NAME
SNOWFLAKE_TEST_PASSWORD: op://github-actions/snowflake/SNOWFLAKE_TEST_PASSWORD
SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: op://github-actions/snowflake/SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE
SNOWFLAKE_TEST_PRIVATE_KEY: op://github-actions/snowflake/SNOWFLAKE_TEST_PRIVATE_KEY
SNOWFLAKE_TEST_SCHEMA_NAME: op://github-actions/snowflake/SNOWFLAKE_TEST_SCHEMA_NAME
SNOWFLAKE_TEST_USER_LOGIN_NAME: op://github-actions/snowflake/SNOWFLAKE_TEST_USER_LOGIN_NAME
SNOWFLAKE_TEST_WAREHOUSE_NAME: op://github-actions/snowflake/SNOWFLAKE_TEST_WAREHOUSE_NAME
- name: Integration Tests (External)
run: nox -s "pytest(ops-external-datastores)"
External-SaaS-Connectors:
needs: Check-Container-Startup
runs-on: ubuntu-latest
timeout-minutes: 30
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
permissions:
contents: read
id-token: write
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Install Nox
run: pip install nox>=2022
- name: Checkout
uses: actions/checkout@v4
- name: Get Vault Token
uses: hashicorp/[email protected]
with:
url: ${{ secrets.VAULT_ADDR }}
namespace: ${{ secrets.VAULT_NAMESPACE }}
method: jwt
role: ${{ secrets.VAULT_ROLE }}
exportToken: True
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_USER }}
password: ${{ env.DOCKER_RO_TOKEN }}
- name: SaaS Connector Tests
env:
VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }}
run: nox -s "pytest(ops-saas)"