merge upstream

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -1,9 +1,29 @@
-*Start with a description of this PR. Then edit the list below to the items that make sense for your PR scope, and check off the boxes as you go!*
+## Summary
 
-## Contributor Checklist
+Major changes:
 
-- [ ] I have broken down my PR scope into the following TODO tasks
-   -  [ ] task 1
-   -  [ ] task 2
+- feature 1: ...
+- fix 1: ...
+
+## Todos
+
+If this is work in progress, what else needs to be done?
+
+- feature 2: ...
+- fix 2:
+
+## Checklist
+
+- [ ] Google format doc strings added.
+- [ ] Code linted with `ruff`. (For guidance in fixing rule violates, see [rule list](https://beta.ruff.rs/docs/rules/))
+- [ ] Type annotations included. Check with `mypy`.
+- [ ] Tests added for new features/fixes.
 - [ ] I have run the tests locally and they passed.
-- [ ] I have added tests, or extended existing tests, to cover any new features or bugs fixed in this PR
+<!-- - [ ] If applicable, new classes/functions/modules have [`duecredit`](https://github.com/duecredit/duecredit) `@due.dcite` decorators to reference relevant papers by DOI ([example](https://github.com/materialsproject/pymatgen/blob/91dbe6ee9ed01d781a9388bf147648e20c6d58e0/pymatgen/core/lattice.py#L1168-L1172)) -->
+
+Tip: Install `pre-commit` hooks to auto-check types and linting before every commit:
+
+```sh
+pip install -U pre-commit
+pre-commit install
+```

.github/release.yml

@@ -0,0 +1,28 @@
+changelog:
+  exclude:
+    authors: [dependabot, github-actions, pre-commit-ci]
+  categories:
+    - title: 🎉 New Features
+      labels: [feature]
+    - title: 🐛 Bug Fixes
+      labels: [fix]
+    - title: 🛠 Enhancements
+      labels: [enhancement, DX, UX]
+    - title: 📖 Documentation
+      labels: [docs]
+    - title: 🧹 House-Keeping
+      labels: [housekeeping]
+    - title: 🚀 Performance
+      labels: [performance]
+    - title: 💡 Refactoring
+      labels: [refactor]
+    - title: 🧪 Tests
+      labels: [tests]
+    - title: 💥 Breaking Changes
+      labels: [breaking]
+    - title: 🔒 Security Fixes
+      labels: [security]
+    - title: 🏥 Package Health
+      labels: [pkg]
+    - title: 🤷‍♂️ Other Changes
+      labels: ["*"]

.github/workflows/codeql.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron: "11 19 * * 6"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
@@ -24,7 +28,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/post-process.yml

@@ -2,24 +2,15 @@ name: Post-process
 
 on:
   workflow_run:
+    branches:
+      - main
     types:
       - completed
     workflows:
       # List all required workflow names here.
       - 'testing'
 
 jobs:
-  merge-me:
-    name: Merge me!
-    runs-on: ubuntu-latest
-    steps:
-      - # It is often a desired behavior to merge only when a workflow execution
-        # succeeds. This can be changed as needed.
-        if: ${{ github.event.workflow_run.conclusion == 'success' }}
-        name: Merge me!
-        uses: ridedott/merge-me-action@v2
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   auto-gen-release:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -9,11 +9,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |
@@ -36,19 +36,17 @@ jobs:
     - deploy
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.7
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-optional.txt
+        pip install -r requirements/ubuntu-latest_py3.10_extras.txt
         pip install -e .
-        pip install -r requirements-docs.txt
 
     - name: Generate changelog
       uses: charmixer/auto-changelog-action@v1
@@ -72,7 +70,7 @@ jobs:
       run: mkdocs build
 
     - name: Deploy
-      uses: peaceiris/actions-gh-pages@v3.9.0
+      uses: peaceiris/actions-gh-pages@v4.0.0
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ./site

.github/workflows/testing.yml

@@ -11,41 +11,28 @@ on:
     branches:
       - main
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     runs-on: ubuntu-latest
-
+    strategy:
+      max-parallel: 1
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-testing.txt
-
-    - name: Lint with pycodestyle
-      run: |
-        pip install pycodestyle
-        pycodestyle src/maggma
-
-    - name: Lint with mypy
-      run: |
-        pip install mypy mypy-boto3
-        mypy src/maggma
-
-    - name: Lint with flake8
-      run: |
-        pip install flake8
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 --count --show-source --statistics src/maggma
-        # exit-zero treats all errors as warnings.
-        flake8 --count --exit-zero --max-complexity=20 --statistics src/maggma
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: pip
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files
 
   test:
     needs: lint
@@ -64,24 +51,22 @@ jobs:
       max-parallel: 6
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-testing.txt
-        pip install -r requirements-optional.txt
+        pip install -r requirements/${{ matrix.os }}_py${{ matrix.python-version }}_extras.txt
 
     - name: Test with pytest
       env:
@@ -90,7 +75,7 @@ jobs:
       run: |
         pip install -e .
         pytest --cov=maggma --cov-report=xml
-    - uses: codecov/codecov-action@v3.1.1
+    - uses: codecov/codecov-action@v4.3.1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml
@@ -99,18 +84,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.8
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e .
-        pip install -r requirements-optional.txt
-        pip install -r requirements-docs.txt
+        pip install -e .[docs]
 
     - name: Build
       run: mkdocs build

.github/workflows/upgrade-dependencies.yml

@@ -0,0 +1,93 @@
+# https://www.oddbird.net/2022/06/01/dependabot-single-pull-request/
+# https://github.com/materialsproject/MPContribs/blob/master/.github/workflows/upgrade-dependencies.yml
+name: upgrade dependencies
+
+on:
+  workflow_dispatch: # Allow running on-demand
+  schedule:
+    # Runs every Monday at 8:00 UTC (4:00 Eastern)
+    - cron: '0 8 * * 1'
+
+jobs:
+  upgrade:
+    name: ${{ matrix.package }} (${{ matrix.os }}/py${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ['ubuntu-latest', 'macos-latest']
+        package: ["."]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Upgrade Python dependencies
+        shell: bash
+        run: |
+          python${{ matrix.python-version }} -m pip install --upgrade pip pip-tools
+          cd ${{ matrix.package }}
+          python${{ matrix.python-version }} -m piptools compile -q --upgrade --resolver=backtracking -o requirements/${{ matrix.os }}_py${{ matrix.python-version }}.txt
+          python${{ matrix.python-version }} -m piptools compile -q --upgrade --resolver=backtracking --all-extras -o requirements/${{ matrix.os }}_py${{ matrix.python-version }}_extras.txt
+      - name: Detect changes
+        id: changes
+        shell: bash
+        run: |
+          #git diff-index HEAD ${{ matrix.package }}/requirements/${{ matrix.os }}_py${{ matrix.python-version }}*.txt | awk '{print $4}' | sort -u
+          #sha1=$(git diff-index HEAD ${{ matrix.package }}/requirements/${{ matrix.os }}_py${{ matrix.python-version }}*.txt | awk '{print $4}' | sort -u | head -n1)
+          #[[ $sha1 == "0000000000000000000000000000000000000000" ]] && git update-index --really-refresh ${{ matrix.package }}/requirements/${{ matrix.os }}_py${{ matrix.python-version }}*.txt
+          echo "count=$(git diff-index HEAD ${{ matrix.package }}/requirements/${{ matrix.os }}_py${{ matrix.python-version }}*.txt | wc -l | xargs)" >> $GITHUB_OUTPUT
+          echo "files=$(git ls-files --exclude-standard --others ${{ matrix.package }}/requirements/${{ matrix.os }}_py${{ matrix.python-version }}*.txt | wc -l | xargs)" >> $GITHUB_OUTPUT
+      - name: commit & push changes
+        if: steps.changes.outputs.count > 0 || steps.changes.outputs.files > 0
+        shell: bash
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add ${{ matrix.package }}/requirements
+          git commit -m "update dependencies for ${{ matrix.package }} (${{ matrix.os }}/py${{ matrix.python-version }})"
+          git push -f origin ${{ github.ref_name }}:auto-dependency-upgrades-${{ matrix.package }}-${{ matrix.os }}-py${{ matrix.python-version }}
+
+  pull_request:
+    name: Merge all branches and open PR
+    runs-on: ubuntu-latest
+    needs: upgrade
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: detect auto-upgrade-dependency branches
+        id: changes
+        run: echo "count=$(git branch -r | grep auto-dependency-upgrades- | wc -l | xargs)" >> $GITHUB_OUTPUT
+      - name: merge all auto-dependency-upgrades branches
+        if: steps.changes.outputs.count > 0
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git checkout -b auto-dependency-upgrades
+          git branch -r | grep auto-dependency-upgrades- | xargs -I {} git merge {}
+          git rebase ${GITHUB_REF##*/}
+          git push -f origin auto-dependency-upgrades
+          git branch -r | grep auto-dependency-upgrades- | cut -d/ -f2 | xargs -I {} git push origin :{}
+      - name: Open pull request if needed
+        if: steps.changes.outputs.count > 0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # Only open a PR if the branch is not attached to an existing one
+        # note that this auto-created PR will not trigger the testing workflow, which
+        # is an intentional limitation imposed by GitHub. See
+        # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+        # The simplest workaround is to close and immediately re-open the Auto PR
+        run: |
+          PR=$(gh pr list --head auto-dependency-upgrades --json number -q '.[0].number')
+          if [ -z $PR ]; then
+            gh pr create \
+            --head auto-dependency-upgrades \
+            --title "Automated dependency upgrades" \
+            --body "Full log: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            echo "Pull request already exists, won't create a new one."
+          fi