[MAINTENANCE] update pip installs in pipelines (great-expectations#6609)

azure-pipelines-contrib.yml

@@ -38,7 +38,7 @@ stages:
             displayName: 'Use Python $(python.version)'
 
           - script: |
-              pip install isort[requirements]==5.10.1 flake8==5.0.4 black==22.3.0 pyupgrade==2.7.2 invoke==1.7.1
+              pip install $(grep -E '^(black|invoke|isort|flake8|pyupgrade)' reqs/requirements-dev-contrib.txt)
               EXIT_STATUS=0
               invoke fmt --check || EXIT_STATUS=$?
               invoke lint || EXIT_STATUS=$?
@@ -71,8 +71,7 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install twine
-              pip install wheel
+              pip install twine wheel
               git config --global user.email "[email protected]"
               git config --global user.name "Great Expectations"
               echo '$(contrib_major).$(contrib_minor).$(Build.BuildNumber)' > ./PACKAGE_VERSION

azure-pipelines-dev.yml

@@ -91,7 +91,7 @@ stages:
             displayName: 'Use Python 3.7'
 
           - script: |
-              pip install isort[requirements]==5.10.1 flake8==5.0.4 black==22.3.0 pyupgrade==2.7.2 invoke==1.7.1
+              pip install $(grep -E '^(black|invoke|isort|flake8|pyupgrade)' reqs/requirements-dev-contrib.txt)
               EXIT_STATUS=0
               invoke fmt --check || EXIT_STATUS=$?
               invoke lint || EXIT_STATUS=$?
@@ -107,7 +107,7 @@ stages:
     - job: type_hint_checker
       steps:
       - script: |
-          pip install --requirement reqs/requirements-dev-contrib.txt
+          pip install $(grep -E '^(invoke|mypy|pytest-cov)' reqs/requirements-dev-contrib.txt)
           invoke type-coverage
         name: TypeHintChecker
 
@@ -126,7 +126,7 @@ stages:
     - job: unused_import_checker
       steps:
       - script: |
-          pip install --requirement reqs/requirements-dev-contrib.txt
+          pip install $(grep -E '^(flake8)' reqs/requirements-dev-contrib.txt)
           # https://www.flake8rules.com/rules/F401.html
           flake8 --select F401 great_expectations tests
         name: UnusedImportChecker
@@ -195,22 +195,10 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-                pip install numpy
-                pip install pandas
-                pip install scipy
-            displayName: 'Install numpy, pandas, and scipy'
-
-          - script: |
-              pip install --requirement requirements.txt
-              echo "about to run pip install --requirement reqs/requirements-dev-test.txt --constraint constraints-dev.txt"
-              pip install --requirement reqs/requirements-dev-test.txt --constraint constraints-dev.txt
-              pip install  .
+              pip install --constraint constraints-dev.txt ".[test]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |
-              # Install dependencies
-              pip install pytest-cov pytest-icdiff pytest-mock pytest-azurepipelines invoke
-
               # Run unit-tests
               invoke tests --ci --cloud --timeout=3.0
 
@@ -276,19 +264,10 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-                pip install pandas
-            displayName: 'Install pandas'
-
-          - script: |
-              pip install --requirement requirements.txt
-              echo "about to run pip install --requirement requirements-dev.txt --constraint constraints-dev.txt"
-              pip install --requirement requirements-dev.txt --constraint constraints-dev.txt
-              pip install  .
+              pip install --constraint constraints-dev.txt ".[dev]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest-cov pytest-azurepipelines
-
               # Run pytest
               pytest $(pytest_args) \
                 --postgresql \
@@ -339,14 +318,11 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install --requirement reqs/requirements-dev-test.txt
-              pip install --requirement requirements.txt
-              pip install .
+              pip install --constraint constraints-dev.txt ".[test]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           # Due to the relatively small number of usage_stats tests, we deem it appropriate to test them in their entirely through pytest
           - script: |
-              pip install pytest-azurepipelines
               pytest --no-sqlalchemy --aws-integration -v tests/integration/usage_statistics
 
             displayName: 'pytest'
@@ -392,18 +368,10 @@ stages:
             displayName: 'Configure mysql'
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt \
-                --constraint constraints-dev.txt
-              pip install --requirement requirements.txt
-              pip install .
+              pip install --constraint constraints-dev.txt ".[test, mysql]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |
-              # Install dependencies
-              pip install pytest-cov pytest-azurepipelines
-
               # Run pytest
               pytest tests/test_definitions tests/expectations \
                 --mysql \
@@ -439,18 +407,10 @@ stages:
               sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt \
-                --constraint constraints-dev.txt
-              pip install --requirement requirements.txt
-              pip install .
+              pip install --constraint constraints-dev.txt ".[test, mssql]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |
-              # Install dependencies
-              pip install pytest-cov pytest-azurepipelines
-
               # Run pytest
               pytest tests/test_definitions tests/expectations \
                 --mssql \
@@ -492,18 +452,10 @@ stages:
             displayName: Wait for database to initialise
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt \
-                --constraint constraints-dev.txt
-              pip install --requirement requirements.txt
-              pip install .
+              pip install --constraint constraints-dev.txt ".[test, trino]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |
-              # Install dependencies
-              pip install pytest-cov pytest-azurepipelines
-
               # Run pytest
               pytest tests/test_definitions tests/expectations \
                 --trino \
@@ -542,13 +494,7 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-spark.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt \
-                --constraint constraints-dev.txt
-              pip install --requirement requirements.txt
-              pip install .
+              pip install --constraint constraints-dev.txt ".[test, postgresql, spark]" pytest-azurepipelines
             displayName: 'Install dependencies'
 
           - script: |

azure-pipelines-docs-integration.yml

@@ -104,17 +104,17 @@ stages:
           displayName: 'Update pip'
 
         - script: |
-            pip install --requirement requirements-dev.txt --constraint constraints-dev.txt .
+            pip install --constraint constraints-dev.txt ".[dev]" pytest-azurepipelines google-cloud-bigquery-storage git+https://github.com/awslabs/aws-glue-libs.git
           displayName: 'Install dependencies'
+
         - task: DownloadSecureFile@1
           name: gcp_authkey
           displayName: 'Download GCS Credentials'
           inputs:
               secureFile: 'superconductive-service-acct_ge-oss-ci-cd.json'
               retryCount: '2'
+
         - script: |
-            # install google cloud storage dependency
-            pip install google-cloud-bigquery-storage
             # this is recommended by the Google documentation for CI/CD https://cloud.google.com/sdk/docs/install#other_installation_options
             curl -sS https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-370.0.0-linux-x86_64.tar.gz > ./google-cloud-sdk-370.0.0-linux-x86_64.tar.gz && tar -xf ./google-cloud-sdk-370.0.0-linux-x86_64.tar.gz
             ./google-cloud-sdk/install.sh --usage-reporting=False --path-update=True --quiet --install-python=False
@@ -133,7 +133,6 @@ stages:
           displayName: 'Create MSSQL database test_ci'
 
         - script: |
-            pip install pytest pytest-azurepipelines git+https://github.com/awslabs/aws-glue-libs.git
             pytest -v --docs-tests -m integration --mysql --bigquery --mssql --spark --postgresql --trino --aws --azure tests/integration/test_script_runner.py
           displayName: 'pytest'
           env:

azure-pipelines-gcp-integration.yml

@@ -26,11 +26,9 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install --requirement requirements-dev.txt --constraint constraints-dev.txt
+              pip install --constraint constraints-dev.txt ".[dev]" pytest-azurepipelines google-cloud-bigquery-storage
             displayName: 'Install dependencies'
-          - script: |
-              pip install google-cloud-bigquery-storage
-            displayName: 'Install Google Cloud dependencies'
+
           - task: DownloadSecureFile@1
             name: gcp_authkey
             displayName: 'Download Google Service Account'
@@ -39,7 +37,6 @@ stages:
               retryCount: '2'
 
           - script: |
-              pip install pytest-azurepipelines
               pip freeze > pip-freeze.txt
               mkdir -p junit
               pytest -v $(test_script) \
@@ -54,6 +51,7 @@ stages:
               GE_TEST_GCP_PROJECT: $(GE_TEST_GCP_PROJECT)
               GE_TEST_BIGQUERY_DATASET: $(GE_TEST_BIGQUERY_DATASET)
               GE_TEST_BIGQUERY_PERFORMANCE_DATASET: $(GE_TEST_BIGQUERY_PERFORMANCE_DATASET)
+
           - task: PublishTestResults@2
             inputs:
               searchFolder: junit
@@ -80,6 +78,7 @@ stages:
               test_script: 'tests/test_definitions/test_expectations_v2_api.py'
               extra_args: ''
           maxParallel: 1
+
         steps:
           - task: UsePythonVersion@0
             inputs:
@@ -90,11 +89,9 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install --requirement requirements-dev.txt --constraint constraints-dev.txt
+              pip install --constraint constraints-dev.txt ".[dev]" pytest-azurepipelines google-cloud-bigquery-storage
             displayName: 'Install dependencies'
-          - script: |
-              pip install google-cloud-bigquery-storage
-            displayName: 'Install Google Cloud dependencies'
+
           - task: DownloadSecureFile@1
             name: gcp_authkey
             displayName: 'Download Google Service Account'
@@ -103,7 +100,6 @@ stages:
               retryCount: '2'
 
           - script: |
-              pip install pytest-azurepipelines
               pip freeze > pip-freeze.txt
               mkdir -p junit
               pytest -v $(test_script) \

azure-pipelines-os-integration.yml

@@ -75,7 +75,6 @@ stages:
   - stage: os_integration
     variables:
       python.version: '3.x'
-      GE_pytest_pip_opts: '--requirement reqs/requirements-dev-test.txt'
       GE_pytest_opts: '--no-sqlalchemy'
 
     jobs:
@@ -94,13 +93,10 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install --requirement requirements.txt
-              pip install $(GE_pytest_pip_opts)
-              pip install .
+              pip install pytest-azurepipeline ".[test]"
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest $(GE_pytest_opts)
             displayName: 'pytest'
 
@@ -118,13 +114,10 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install --requirement requirements.txt
-              pip install $(GE_pytest_pip_opts)
-              pip install .
+              pip install pytest-azurepipeline ".[test]"
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest $(GE_pytest_opts)
             displayName: 'pytest'
 
@@ -165,18 +158,12 @@ stages:
             displayName: 'Configure mysql'
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt
+              pip install ".[test, mysql]" pytest-azurepipelines
               # Install latest sqlalchemy version
               pip install --upgrade SQLAlchemy
-              pip install --requirement requirements.txt
-              pip install .
             displayName: 'Install dependencies'
 
           - script: |
-              pip install --requirement requirements.txt
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest --mysql --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
             displayName: 'pytest'
 
@@ -202,17 +189,12 @@ stages:
               sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt
+              pip install ".[test, mssql]" pytest-azurepipelines
               # Install latest sqlalchemy version
               pip install --upgrade SQLAlchemy
-              pip install --requirement requirements.txt
-              pip install .
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest --mssql --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
             displayName: 'pytest'
 
@@ -235,17 +217,12 @@ stages:
             displayName: 'Update pip'
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt
+              pip install ".[test, postgresql]" pytest-azurepipelines
               # Install latest sqlalchemy version
               pip install --upgrade SQLAlchemy
-              pip install --requirement requirements.txt
-              pip install .
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest --postgresql --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
             displayName: 'pytest'
 
@@ -277,16 +254,11 @@ stages:
             displayName: Wait for database to initialise
 
           - script: |
-              pip install \
-                --requirement reqs/requirements-dev-test.txt \
-                --requirement reqs/requirements-dev-sqlalchemy.txt
+              pip install ".[test, trino]" pytest-azurepipelines
               # Install latest sqlalchemy version
               pip install --upgrade SQLAlchemy
-              pip install --requirement requirements.txt
-              pip install .
             displayName: 'Install dependencies'
 
           - script: |
-              pip install pytest pytest-cov pytest-azurepipelines
               pytest --trino --napoleon-docstrings --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html --ignore=tests/cli --ignore=tests/integration/usage_statistics
             displayName: 'pytest'