From 7177d61e42ddba542829c6c057681cb3fd7904e9 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 30 Jan 2023 08:16:40 -0500 Subject: [PATCH 1/3] don't use `crds sync` for CRDS caching --- .github/workflows/ci.yml | 67 ++++++++++++++--------------------- .github/workflows/ci_cron.yml | 32 +++++++---------- 2 files changed, 39 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7f801d14b..4e67f95d0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,47 +21,8 @@ env: CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 jobs: - check: - name: ${{ matrix.toxenv }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - toxenv: [ check-style, check-security, check-dependencies, build-dist ] - python-version: [ '3.x' ] - os: [ ubuntu-latest ] - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: 'setup.cfg' - - run: pip install tox - - run: tox -e ${{ matrix.toxenv }} - cache_crds: - name: cache CRDS files - runs-on: ubuntu-latest - outputs: - crds-context: ${{ steps.crds-context.outputs.pmap }} - steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - run: pip install crds - - run: echo "pmap=$(crds list --operational-context)" >> $GITHUB_OUTPUT - id: crds-context - - uses: actions/cache@v3 - with: - path: ${{ env.CRDS_PATH }} - key: crds-${{ steps.crds-context.outputs.pmap }} - - run: crds sync --contexts ${{ steps.crds-context.outputs.pmap }} - if: ${{ steps.crds-context.outputs.pmap != '' }} test: name: ${{ matrix.toxenv }} (Python ${{ matrix.python-version }}, ${{ matrix.os }}) - needs: [ cache_crds ] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -70,6 +31,18 @@ jobs: python-version: [ '3.8', '3.9', '3.10', '3.11' ] os: [ ubuntu-latest, macos-latest ] include: + - toxenv: check-style + os: ubuntu-latest + python-version: '3.x' + - toxenv: check-security + os: ubuntu-latest + python-version: '3.x' + - toxenv: check-dependencies + os: ubuntu-latest + python-version: '3.x' + - toxenv: build-dist + os: ubuntu-latest + python-version: '3.x' - toxenv: test-cov-xdist os: ubuntu-latest python-version: '3.11' @@ -94,11 +67,23 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' cache-dependency-path: 'setup.cfg' - - run: pip install tox + - run: > + echo "pmap=$( + curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | + python -c "import sys, json; print(json.load(sys.stdin)['result'])" + )" >> $GITHUB_OUTPUT + # Get default CRDS_CONTEXT without installing crds client + # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request + id: crds-context + if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} + - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" + if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} - uses: actions/cache@v3 with: path: ${{ env.CRDS_PATH }} - key: crds-${{ needs.cache_crds.outputs.crds-context }} + key: crds-${{ steps.crds-context.outputs.pmap }} + if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} + - run: pip install tox - run: tox -e ${{ matrix.toxenv }} - if: ${{ contains(matrix.toxenv, '-cov') }} uses: codecov/codecov-action@v3 diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 4a08833a34..be6fb5b9b4 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -19,36 +19,30 @@ jobs: strategy: fail-fast: false matrix: - toxenv: [test-devdeps-xdist] - python-version: ['3.x'] - os: [ubuntu-latest] + toxenv: [ test-devdeps-xdist ] + python-version: [ '3.x' ] + os: [ ubuntu-latest ] steps: - - name: Checkout code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: setup.cfg - - name: Get CRDS context - id: crds-context - # Get default CRDS_CONTEXT without installing crds client - # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request - run: > + cache-dependency-path: 'setup.cfg' + - run: > echo "pmap=$( curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' https://jwst-crds.stsci.edu/json/ | python -c "import sys, json; print(json.load(sys.stdin)['result'])" )" >> $GITHUB_OUTPUT + # Get default CRDS_CONTEXT without installing crds client + # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request + id: crds-context - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" - - name: Cache CRDS reference files - uses: actions/cache@v3 + - uses: actions/cache@v3 with: path: ${{ env.CRDS_PATH }} key: crds-reffiles-${{ steps.crds-context.outputs.pmap }} - - name: Run ${{ matrix.toxenv }} - run: | - pip install tox - tox -e ${{ matrix.toxenv }} + - run: pip install tox + - run: tox -e ${{ matrix.toxenv }} From 1ceb7342d77f14327eece429db9f1e267874d75d Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 30 Jan 2023 08:45:25 -0500 Subject: [PATCH 2/3] put `if:` at the beginning of block --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e67f95d0d..ad94587279 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,8 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' cache-dependency-path: 'setup.cfg' - - run: > + - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} + run: > echo "pmap=$( curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ | python -c "import sys, json; print(json.load(sys.stdin)['result'])" @@ -75,14 +76,13 @@ jobs: # Get default CRDS_CONTEXT without installing crds client # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request id: crds-context - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} - - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} - - uses: actions/cache@v3 + - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} + run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" + - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} + uses: actions/cache@v3 with: path: ${{ env.CRDS_PATH }} key: crds-${{ steps.crds-context.outputs.pmap }} - if: ${{ contains(matrix.toxenv, 'test') && !contains(matrix.toxenv, 'opencv') }} - run: pip install tox - run: tox -e ${{ matrix.toxenv }} - if: ${{ contains(matrix.toxenv, '-cov') }} From d8b42f17ed3c69fecdcfa50f69c526f97e7f1ad2 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 30 Jan 2023 13:12:13 -0500 Subject: [PATCH 3/3] use `/tmp` instead of `$HOME` for CRDS path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad94587279..813a3830d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: env: CRDS_SERVER_URL: https://jwst-crds.stsci.edu - CRDS_PATH: $HOME/crds_cache + CRDS_PATH: /tmp/crds_cache CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20