Better handling and documentation of dependencies

.github/workflows/tests.yml → .github/workflows/ci.yml

@@ -1,13 +1,16 @@
-name: Tests
+name: CI Tests
 
-on: [pull_request, push]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -16,45 +19,54 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
-    - name: sphinx-build
+    - name: sphinx documentation
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         make -C doc html SPHINXOPTS="-W --keep-going"
-    - name: pytest unit-tests & cov-report
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
         pytest --cov=improver --cov-report xml:coverage.xml
-    - name: codacy-coverage
-      if: env.CODACY_PROJECT_TOKEN
+    - name: codacy upload
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         python-codacy-coverage -v -r coverage.xml
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
-    - uses: codecov/codecov-action@v1
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
   Codestyle-and-flake8:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30]
+        env: [environment_a]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -63,41 +75,41 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: isort
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         isort --check-only .
     - name: black
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         black --check .
     - name: flake8
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         flake8 improver improver_tests
   Safety-Bandit:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -106,60 +118,27 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: safety
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         safety check || true
     - name: bandit
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         bandit -r improver
-  Type-checking:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        env: [py37_iris30]
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      id: cache
-      env:
-        # Increase this value to reset cache
-        CACHE_NUMBER: 2
-      with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
-    - name: conda env update
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
-    - name: conda info
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
-        conda info
-        conda list
-    - name: mypy
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
-        mypy improver || true

.github/workflows/scheduled.yml

@@ -0,0 +1,137 @@
+name: Scheduled Tests
+
+on:
+  schedule:
+    - cron: '7 4 * * *'
+  workflow_dispatch:
+jobs:
+  Sphinx-Pytest-Coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: sphinx documentation
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        make -C doc html SPHINXOPTS="-W --keep-going"
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest --cov=improver --cov-report xml:coverage.xml
+    - name: codacy upload
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        python-codacy-coverage -v -r coverage.xml
+      env:
+        CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
+  Safety-Bandit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: safety
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        safety check || true
+    - name: bandit
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        bandit -r improver
+  Type-checking:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: mypy
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        mypy improver || true