From b15826f0dab041e4716355bca21090f78a9f00f5 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:24:02 +0200 Subject: [PATCH 01/23] Add some more basic checks to pre-commit --- .pre-commit-config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 864ed4d..2fe9e1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,14 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace + - id: check-merge-conflict + - id: mixed-line-ending + - id: check-added-large-files + args: ["--maxkb=2000"] - repo: https://github.com/psf/black rev: 22.8.0 From 1ea91d0764a9c49606b9f0a20012b1166ac6a73d Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:24:15 +0200 Subject: [PATCH 02/23] Add release instructions to README --- README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c03dd2d..f51289f 100644 --- a/README.md +++ b/README.md @@ -132,14 +132,11 @@ VS Code will highlight the changes in the two files, which should correspond to ### Publishing the Tool -To publish a new version of the tool on PyPI, update the version number in `pyproject.toml`, and then run: -```bash -python -m pip install --upgrade build -python -m pip install --upgrade twine -rm -rf dist -python -m build -python -m twine upload dist/* -``` +Follow these steps to release a new version of `xl2times` and publish it on PyPI: +- [ ] Bump the version number in `pyproject.toml` (use [Semantic Versioning](https://semver.org/)) +- [ ] Open a PR with this change titled "Release vX.Y.Z" +- [ ] When the PR is merged, create a [new release](https://github.com/etsap-TIMES/xl2times/releases/new) titled "vX.Y.Z". Select "Create a new tag: on publish" and click "Generate release notes" to generate the notes automatically. +- [ ] Click "Publish release" to publish the release on GitHub. A GitHub Actions workflow will automatically upload the distribution to PyPI. ## Contributing From c13ad78096b8eb8638450a2298dd59663b7ca8a9 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:33:35 +0200 Subject: [PATCH 03/23] CI: Publish to TestPyPI --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9aed4b..2747988 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,3 +170,20 @@ jobs: with: path: ~/.cache/xl2times key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} + + # ---------- Build package and upload to PyPI + + - name: Build package + working-directory: xl2times + run: | + source .venv/bin/activate + python -m pip install --upgrade build twine + rm -rf dist || true + python -m build + + - name: Download dist built by CI job + # if: startsWith(github.ref, 'refs/tags/v') # Only run on tags `v...` + # if: github.event_name == 'release' && github.event.action == 'created' # TODO use this before merge + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ From a74fe98ba4eae702537d0eef4c1fa853441ec447 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:42:15 +0200 Subject: [PATCH 04/23] Fix publish permission; version to alpha for now --- .github/workflows/ci.yml | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2747988..c0a42e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ on: jobs: CI: runs-on: ubuntu-latest + permissions: + id-token: write env: PY_VERSION: "3.11" @@ -181,7 +183,7 @@ jobs: rm -rf dist || true python -m build - - name: Download dist built by CI job + - name: Publish to PyPI # if: startsWith(github.ref, 'refs/tags/v') # Only run on tags `v...` # if: github.event_name == 'release' && github.event.action == 'created' # TODO use this before merge uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml index d8df927..abf36db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ packages = ["xl2times"] [project] name = "xl2times" -version = "0.1.0" +version = "0.1.0-alpha" authors = [ { name="Sam Webster", email="13457618+samwebster@users.noreply.github.com" }, { name="Tom Minka", email="8955276+tminka@users.noreply.github.com" }, From 8ae889ed249a27f8ed71461cf0ecb11b6c05509d Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:50:26 +0200 Subject: [PATCH 05/23] Fix publish directory --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0a42e0..2bc5fa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,3 +189,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + packages-dir: xl2times/custom-dir/ From 142dbffb9dbc4ebae59f4a493661d1e962b376b3 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:50:42 +0200 Subject: [PATCH 06/23] Temporarily disable tests --- .github/workflows/ci.yml | 258 +++++++++++++++++++-------------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bc5fa4..46ea1e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,135 +43,135 @@ jobs: python -m pip install --upgrade pip python -m pip install -e .[dev] - - name: Check code formatting - working-directory: xl2times - run: | - source .venv/bin/activate - pre-commit install - pre-commit run --all-files - - - name: Run unit tests - working-directory: xl2times - run: | - source .venv/bin/activate - pytest - - # ---------- Prepare ETSAP Demo models - - - uses: actions/checkout@v3 - with: - repository: etsap-TIMES/TIMES_model - path: TIMES_model - ref: ${{ env.REF_TIMES_model }} - - - uses: actions/checkout@v3 - with: - repository: olejandro/demos-dd - path: xl2times/benchmarks/dd - ref: ${{ env.REF_demos-dd }} - - - uses: actions/checkout@v3 - with: - repository: olejandro/demos-xlsx - path: xl2times/benchmarks/xlsx - ref: ${{ env.REF_demos-xlsx }} - token: ${{ secrets.GH_PAT_DEMOS_XLSX }} - - # ---------- Prepare TIMES Ireland Model - - # We add this model as the directory `ireland` under `benchmarks/{xlsx,dd}/` - # so that the run_benchmarks.py script runs this model too - - uses: actions/checkout@v3 - with: - repository: esma-cgep/tim - path: xl2times/benchmarks/xlsx/Ireland - ref: ${{ env.REF_tim }} - - - uses: actions/checkout@v3 - with: - repository: esma-cgep/tim-gams - path: xl2times/benchmarks/dd/Ireland - ref: ${{ env.REF_tim-gams }} - - # ---------- Install GAMS - - - name: Install GAMS - env: - GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - if: ${{ env.GAMS_LICENSE != '' }} - run: | - curl https://d37drm4t2jghv5.cloudfront.net/distributions/44.1.0/linux/linux_x64_64_sfx.exe -o linux_x64_64_sfx.exe - chmod +x linux_x64_64_sfx.exe - mkdir GAMS - pushd GAMS - ../linux_x64_64_sfx.exe > /dev/null && echo Successfully installed GAMS - export PATH=$PATH:$(pwd)/gams44.1_linux_x64_64_sfx - popd - echo Creating license file at $HOME/.local/share/GAMS - mkdir -p $HOME/.local/share/GAMS - echo "$GAMS_LICENSE" > $HOME/.local/share/GAMS/gamslice.txt - ls -l $HOME/.local/share/GAMS/ - - # ---------- Run tool, check for regressions - - - name: Restore XLSX cache directory from cache - id: cache - uses: actions/cache/restore@v4 - with: - path: ~/.cache/xl2times - # Cache key is refs of the input xlsx repos, since that's what is cached - key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} - # If we can't find the exact key for the TIM repo, still use the cache if the demos repo ref matches - restore-keys: | - ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}- - ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}- - ${{ runner.os }}-py-${{ env.PY_VERSION }}- - - - name: Run tool on all benchmarks - env: - GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - if: ${{ env.GAMS_LICENSE != '' }} - working-directory: xl2times - # Use tee to also save the output to out.txt so that the summary table can be - # printed again in the next step. - # Save the return code to retcode.txt so that the next step can fail the action - run: | - source .venv/bin/activate - export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx - (python utils/run_benchmarks.py benchmarks.yml \ - --dd --times_dir $GITHUB_WORKSPACE/TIMES_model \ - --verbose \ - | tee out.txt; \ - echo ${PIPESTATUS[0]} > retcode.txt) - - - name: Run CSV-only regression tests (no GAMS license) - env: - GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - if: ${{ env.GAMS_LICENSE == '' }} - working-directory: xl2times - # Run without --dd flag if GAMS license secret doesn't exist. - # Useful for testing for (CSV) regressions in forks before creating PRs. - run: | - source .venv/bin/activate - export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx - (python utils/run_benchmarks.py benchmarks.yml \ - --times_dir $GITHUB_WORKSPACE/TIMES_model \ - --verbose \ - | tee out.txt; \ - echo ${PIPESTATUS[0]} > retcode.txt) - - - name: Print summary - working-directory: xl2times - run: | - sed -n '/Benchmark *Time.*Accuracy/h;//!H;$!d;x;//p' out.txt - exit $(cat retcode.txt) - - - uses: actions/cache/save@v4 - # Save the cache even if the regression tests fail - if: always() && !steps.cache-restore.outputs.cache-hit - with: - path: ~/.cache/xl2times - key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} + # - name: Check code formatting + # working-directory: xl2times + # run: | + # source .venv/bin/activate + # pre-commit install + # pre-commit run --all-files + + # - name: Run unit tests + # working-directory: xl2times + # run: | + # source .venv/bin/activate + # pytest + + # # ---------- Prepare ETSAP Demo models + + # - uses: actions/checkout@v3 + # with: + # repository: etsap-TIMES/TIMES_model + # path: TIMES_model + # ref: ${{ env.REF_TIMES_model }} + + # - uses: actions/checkout@v3 + # with: + # repository: olejandro/demos-dd + # path: xl2times/benchmarks/dd + # ref: ${{ env.REF_demos-dd }} + + # - uses: actions/checkout@v3 + # with: + # repository: olejandro/demos-xlsx + # path: xl2times/benchmarks/xlsx + # ref: ${{ env.REF_demos-xlsx }} + # token: ${{ secrets.GH_PAT_DEMOS_XLSX }} + + # # ---------- Prepare TIMES Ireland Model + + # # We add this model as the directory `ireland` under `benchmarks/{xlsx,dd}/` + # # so that the run_benchmarks.py script runs this model too + # - uses: actions/checkout@v3 + # with: + # repository: esma-cgep/tim + # path: xl2times/benchmarks/xlsx/Ireland + # ref: ${{ env.REF_tim }} + + # - uses: actions/checkout@v3 + # with: + # repository: esma-cgep/tim-gams + # path: xl2times/benchmarks/dd/Ireland + # ref: ${{ env.REF_tim-gams }} + + # # ---------- Install GAMS + + # - name: Install GAMS + # env: + # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + # if: ${{ env.GAMS_LICENSE != '' }} + # run: | + # curl https://d37drm4t2jghv5.cloudfront.net/distributions/44.1.0/linux/linux_x64_64_sfx.exe -o linux_x64_64_sfx.exe + # chmod +x linux_x64_64_sfx.exe + # mkdir GAMS + # pushd GAMS + # ../linux_x64_64_sfx.exe > /dev/null && echo Successfully installed GAMS + # export PATH=$PATH:$(pwd)/gams44.1_linux_x64_64_sfx + # popd + # echo Creating license file at $HOME/.local/share/GAMS + # mkdir -p $HOME/.local/share/GAMS + # echo "$GAMS_LICENSE" > $HOME/.local/share/GAMS/gamslice.txt + # ls -l $HOME/.local/share/GAMS/ + + # # ---------- Run tool, check for regressions + + # - name: Restore XLSX cache directory from cache + # id: cache + # uses: actions/cache/restore@v4 + # with: + # path: ~/.cache/xl2times + # # Cache key is refs of the input xlsx repos, since that's what is cached + # key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} + # # If we can't find the exact key for the TIM repo, still use the cache if the demos repo ref matches + # restore-keys: | + # ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}- + # ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}- + # ${{ runner.os }}-py-${{ env.PY_VERSION }}- + + # - name: Run tool on all benchmarks + # env: + # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + # if: ${{ env.GAMS_LICENSE != '' }} + # working-directory: xl2times + # # Use tee to also save the output to out.txt so that the summary table can be + # # printed again in the next step. + # # Save the return code to retcode.txt so that the next step can fail the action + # run: | + # source .venv/bin/activate + # export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx + # (python utils/run_benchmarks.py benchmarks.yml \ + # --dd --times_dir $GITHUB_WORKSPACE/TIMES_model \ + # --verbose \ + # | tee out.txt; \ + # echo ${PIPESTATUS[0]} > retcode.txt) + + # - name: Run CSV-only regression tests (no GAMS license) + # env: + # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + # if: ${{ env.GAMS_LICENSE == '' }} + # working-directory: xl2times + # # Run without --dd flag if GAMS license secret doesn't exist. + # # Useful for testing for (CSV) regressions in forks before creating PRs. + # run: | + # source .venv/bin/activate + # export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx + # (python utils/run_benchmarks.py benchmarks.yml \ + # --times_dir $GITHUB_WORKSPACE/TIMES_model \ + # --verbose \ + # | tee out.txt; \ + # echo ${PIPESTATUS[0]} > retcode.txt) + + # - name: Print summary + # working-directory: xl2times + # run: | + # sed -n '/Benchmark *Time.*Accuracy/h;//!H;$!d;x;//p' out.txt + # exit $(cat retcode.txt) + + # - uses: actions/cache/save@v4 + # # Save the cache even if the regression tests fail + # if: always() && !steps.cache-restore.outputs.cache-hit + # with: + # path: ~/.cache/xl2times + # key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} # ---------- Build package and upload to PyPI From cdffc06caacb64675a05b9cf9f0501a691778ede Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 08:57:44 +0200 Subject: [PATCH 07/23] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ea1e0..7f682d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,4 +189,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ - packages-dir: xl2times/custom-dir/ + packages-dir: xl2times/dist/ From 536032ca2a59c3cf8173dac1fb9d9e2fc8702a5b Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 09:11:37 +0200 Subject: [PATCH 08/23] Change version just for test pypi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index abf36db..3b20d1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ packages = ["xl2times"] [project] name = "xl2times" -version = "0.1.0-alpha" +version = "0.1.0-alpha.1" authors = [ { name="Sam Webster", email="13457618+samwebster@users.noreply.github.com" }, { name="Tom Minka", email="8955276+tminka@users.noreply.github.com" }, From 79dd7607ba72318a97847dbaeffaeae9eb597993 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 09:38:44 +0200 Subject: [PATCH 09/23] Fix package discovery --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3b20d1b..a43bb73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,9 @@ requires = ["setuptools>=61.0.0", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = ["xl2times"] +[tool.setuptools.packages.find] +where = [""] +include = ["xl2times", "xl2times.*"] [project] name = "xl2times" From c2a3a67c599591937641eb30cd2ddeb390412b65 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sat, 30 Mar 2024 09:39:24 +0200 Subject: [PATCH 10/23] Also bump __init__.py version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f51289f..685dbaa 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ VS Code will highlight the changes in the two files, which should correspond to ### Publishing the Tool Follow these steps to release a new version of `xl2times` and publish it on PyPI: -- [ ] Bump the version number in `pyproject.toml` (use [Semantic Versioning](https://semver.org/)) +- [ ] Bump the version number in `pyproject.toml` and `xl2times/__init__.py` (use [Semantic Versioning](https://semver.org/)) - [ ] Open a PR with this change titled "Release vX.Y.Z" - [ ] When the PR is merged, create a [new release](https://github.com/etsap-TIMES/xl2times/releases/new) titled "vX.Y.Z". Select "Create a new tag: on publish" and click "Generate release notes" to generate the notes automatically. - [ ] Click "Publish release" to publish the release on GitHub. A GitHub Actions workflow will automatically upload the distribution to PyPI. From fdfea5beb8f2d12b6af8a6bb8bd5925fdc5881c7 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:18:44 +0200 Subject: [PATCH 11/23] Use built whl in CI to test package uploaded to PyPI --- .github/workflows/ci.yml | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f682d3..b88932a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,20 +35,23 @@ jobs: with: python-version: ${{ env.PY_VERSION }} - - name: Install tool and dependencies + - name: Check code formatting + working-directory: xl2times + run: | + python -m pip install --upgrade pre-commit + pre-commit install + pre-commit run --all-files + + - name: Build and install xl2times working-directory: xl2times run: | python -m venv .venv source .venv/bin/activate - python -m pip install --upgrade pip - python -m pip install -e .[dev] - - # - name: Check code formatting - # working-directory: xl2times - # run: | - # source .venv/bin/activate - # pre-commit install - # pre-commit run --all-files + python -m pip install --upgrade pip build twine + rm -rf dist || true + python -m build + # Install the built wheel file to imitiate users installing from PyPI: + python -m pip install --no-index --find-links=dist xl2times # - name: Run unit tests # working-directory: xl2times @@ -173,15 +176,7 @@ jobs: # path: ~/.cache/xl2times # key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} - # ---------- Build package and upload to PyPI - - - name: Build package - working-directory: xl2times - run: | - source .venv/bin/activate - python -m pip install --upgrade build twine - rm -rf dist || true - python -m build + # ---------- Upload package to PyPI on release - name: Publish to PyPI # if: startsWith(github.ref, 'refs/tags/v') # Only run on tags `v...` From c3e765fb6000ca4c132d83f4a7086dec4633014e Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:19:56 +0200 Subject: [PATCH 12/23] Bring back CI tests --- .github/workflows/ci.yml | 244 +++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b88932a..4a7813d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,128 +53,128 @@ jobs: # Install the built wheel file to imitiate users installing from PyPI: python -m pip install --no-index --find-links=dist xl2times - # - name: Run unit tests - # working-directory: xl2times - # run: | - # source .venv/bin/activate - # pytest - - # # ---------- Prepare ETSAP Demo models - - # - uses: actions/checkout@v3 - # with: - # repository: etsap-TIMES/TIMES_model - # path: TIMES_model - # ref: ${{ env.REF_TIMES_model }} - - # - uses: actions/checkout@v3 - # with: - # repository: olejandro/demos-dd - # path: xl2times/benchmarks/dd - # ref: ${{ env.REF_demos-dd }} - - # - uses: actions/checkout@v3 - # with: - # repository: olejandro/demos-xlsx - # path: xl2times/benchmarks/xlsx - # ref: ${{ env.REF_demos-xlsx }} - # token: ${{ secrets.GH_PAT_DEMOS_XLSX }} - - # # ---------- Prepare TIMES Ireland Model - - # # We add this model as the directory `ireland` under `benchmarks/{xlsx,dd}/` - # # so that the run_benchmarks.py script runs this model too - # - uses: actions/checkout@v3 - # with: - # repository: esma-cgep/tim - # path: xl2times/benchmarks/xlsx/Ireland - # ref: ${{ env.REF_tim }} - - # - uses: actions/checkout@v3 - # with: - # repository: esma-cgep/tim-gams - # path: xl2times/benchmarks/dd/Ireland - # ref: ${{ env.REF_tim-gams }} - - # # ---------- Install GAMS - - # - name: Install GAMS - # env: - # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - # if: ${{ env.GAMS_LICENSE != '' }} - # run: | - # curl https://d37drm4t2jghv5.cloudfront.net/distributions/44.1.0/linux/linux_x64_64_sfx.exe -o linux_x64_64_sfx.exe - # chmod +x linux_x64_64_sfx.exe - # mkdir GAMS - # pushd GAMS - # ../linux_x64_64_sfx.exe > /dev/null && echo Successfully installed GAMS - # export PATH=$PATH:$(pwd)/gams44.1_linux_x64_64_sfx - # popd - # echo Creating license file at $HOME/.local/share/GAMS - # mkdir -p $HOME/.local/share/GAMS - # echo "$GAMS_LICENSE" > $HOME/.local/share/GAMS/gamslice.txt - # ls -l $HOME/.local/share/GAMS/ - - # # ---------- Run tool, check for regressions - - # - name: Restore XLSX cache directory from cache - # id: cache - # uses: actions/cache/restore@v4 - # with: - # path: ~/.cache/xl2times - # # Cache key is refs of the input xlsx repos, since that's what is cached - # key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} - # # If we can't find the exact key for the TIM repo, still use the cache if the demos repo ref matches - # restore-keys: | - # ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}- - # ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}- - # ${{ runner.os }}-py-${{ env.PY_VERSION }}- - - # - name: Run tool on all benchmarks - # env: - # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - # if: ${{ env.GAMS_LICENSE != '' }} - # working-directory: xl2times - # # Use tee to also save the output to out.txt so that the summary table can be - # # printed again in the next step. - # # Save the return code to retcode.txt so that the next step can fail the action - # run: | - # source .venv/bin/activate - # export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx - # (python utils/run_benchmarks.py benchmarks.yml \ - # --dd --times_dir $GITHUB_WORKSPACE/TIMES_model \ - # --verbose \ - # | tee out.txt; \ - # echo ${PIPESTATUS[0]} > retcode.txt) - - # - name: Run CSV-only regression tests (no GAMS license) - # env: - # GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} - # if: ${{ env.GAMS_LICENSE == '' }} - # working-directory: xl2times - # # Run without --dd flag if GAMS license secret doesn't exist. - # # Useful for testing for (CSV) regressions in forks before creating PRs. - # run: | - # source .venv/bin/activate - # export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx - # (python utils/run_benchmarks.py benchmarks.yml \ - # --times_dir $GITHUB_WORKSPACE/TIMES_model \ - # --verbose \ - # | tee out.txt; \ - # echo ${PIPESTATUS[0]} > retcode.txt) - - # - name: Print summary - # working-directory: xl2times - # run: | - # sed -n '/Benchmark *Time.*Accuracy/h;//!H;$!d;x;//p' out.txt - # exit $(cat retcode.txt) - - # - uses: actions/cache/save@v4 - # # Save the cache even if the regression tests fail - # if: always() && !steps.cache-restore.outputs.cache-hit - # with: - # path: ~/.cache/xl2times - # key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} + - name: Run unit tests + working-directory: xl2times + run: | + source .venv/bin/activate + pytest + + # ---------- Prepare ETSAP Demo models + + - uses: actions/checkout@v3 + with: + repository: etsap-TIMES/TIMES_model + path: TIMES_model + ref: ${{ env.REF_TIMES_model }} + + - uses: actions/checkout@v3 + with: + repository: olejandro/demos-dd + path: xl2times/benchmarks/dd + ref: ${{ env.REF_demos-dd }} + + - uses: actions/checkout@v3 + with: + repository: olejandro/demos-xlsx + path: xl2times/benchmarks/xlsx + ref: ${{ env.REF_demos-xlsx }} + token: ${{ secrets.GH_PAT_DEMOS_XLSX }} + + # ---------- Prepare TIMES Ireland Model + + # We add this model as the directory `ireland` under `benchmarks/{xlsx,dd}/` + # so that the run_benchmarks.py script runs this model too + - uses: actions/checkout@v3 + with: + repository: esma-cgep/tim + path: xl2times/benchmarks/xlsx/Ireland + ref: ${{ env.REF_tim }} + + - uses: actions/checkout@v3 + with: + repository: esma-cgep/tim-gams + path: xl2times/benchmarks/dd/Ireland + ref: ${{ env.REF_tim-gams }} + + # ---------- Install GAMS + + - name: Install GAMS + env: + GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + if: ${{ env.GAMS_LICENSE != '' }} + run: | + curl https://d37drm4t2jghv5.cloudfront.net/distributions/44.1.0/linux/linux_x64_64_sfx.exe -o linux_x64_64_sfx.exe + chmod +x linux_x64_64_sfx.exe + mkdir GAMS + pushd GAMS + ../linux_x64_64_sfx.exe > /dev/null && echo Successfully installed GAMS + export PATH=$PATH:$(pwd)/gams44.1_linux_x64_64_sfx + popd + echo Creating license file at $HOME/.local/share/GAMS + mkdir -p $HOME/.local/share/GAMS + echo "$GAMS_LICENSE" > $HOME/.local/share/GAMS/gamslice.txt + ls -l $HOME/.local/share/GAMS/ + + # ---------- Run tool, check for regressions + + - name: Restore XLSX cache directory from cache + id: cache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/xl2times + # Cache key is refs of the input xlsx repos, since that's what is cached + key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} + # If we can't find the exact key for the TIM repo, still use the cache if the demos repo ref matches + restore-keys: | + ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}- + ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}- + ${{ runner.os }}-py-${{ env.PY_VERSION }}- + + - name: Run tool on all benchmarks + env: + GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + if: ${{ env.GAMS_LICENSE != '' }} + working-directory: xl2times + # Use tee to also save the output to out.txt so that the summary table can be + # printed again in the next step. + # Save the return code to retcode.txt so that the next step can fail the action + run: | + source .venv/bin/activate + export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx + (python utils/run_benchmarks.py benchmarks.yml \ + --dd --times_dir $GITHUB_WORKSPACE/TIMES_model \ + --verbose \ + | tee out.txt; \ + echo ${PIPESTATUS[0]} > retcode.txt) + + - name: Run CSV-only regression tests (no GAMS license) + env: + GAMS_LICENSE: ${{ secrets.GAMS_LICENSE }} + if: ${{ env.GAMS_LICENSE == '' }} + working-directory: xl2times + # Run without --dd flag if GAMS license secret doesn't exist. + # Useful for testing for (CSV) regressions in forks before creating PRs. + run: | + source .venv/bin/activate + export PATH=$PATH:$GITHUB_WORKSPACE/GAMS/gams44.1_linux_x64_64_sfx + (python utils/run_benchmarks.py benchmarks.yml \ + --times_dir $GITHUB_WORKSPACE/TIMES_model \ + --verbose \ + | tee out.txt; \ + echo ${PIPESTATUS[0]} > retcode.txt) + + - name: Print summary + working-directory: xl2times + run: | + sed -n '/Benchmark *Time.*Accuracy/h;//!H;$!d;x;//p' out.txt + exit $(cat retcode.txt) + + - uses: actions/cache/save@v4 + # Save the cache even if the regression tests fail + if: always() && !steps.cache-restore.outputs.cache-hit + with: + path: ~/.cache/xl2times + key: ${{ runner.os }}-py-${{ env.PY_VERSION }}-${{ env.REF_demos-xlsx }}-${{ env.REF_tim }}-${{ env.CACHE_KEY }} # ---------- Upload package to PyPI on release From 29bc93ebbc03d5f53101a209b57e21b57e13fca6 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:39:03 +0200 Subject: [PATCH 13/23] Set publish action to happen only on release --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a7813d..3279b9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,8 @@ on: branches: [main] pull_request: branches: [main] + release: + types: [published] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -179,8 +181,7 @@ jobs: # ---------- Upload package to PyPI on release - name: Publish to PyPI - # if: startsWith(github.ref, 'refs/tags/v') # Only run on tags `v...` - # if: github.event_name == 'release' && github.event.action == 'created' # TODO use this before merge + if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ From ddd882748ea76a9b45295b94c1b0efaae778755f Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:43:23 +0200 Subject: [PATCH 14/23] Move pre-commit check back to after build & install to fix pyright --- .github/workflows/ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3279b9a..8aa4eab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,24 +37,26 @@ jobs: with: python-version: ${{ env.PY_VERSION }} - - name: Check code formatting - working-directory: xl2times - run: | - python -m pip install --upgrade pre-commit - pre-commit install - pre-commit run --all-files - - name: Build and install xl2times working-directory: xl2times run: | python -m venv .venv source .venv/bin/activate - python -m pip install --upgrade pip build twine + python -m pip install --upgrade pip build rm -rf dist || true python -m build # Install the built wheel file to imitiate users installing from PyPI: python -m pip install --no-index --find-links=dist xl2times + - name: Check code formatting + working-directory: xl2times + # Run this step after install so that pyright can find dependencies like pandas + run: | + source .venv/bin/activate + python -m pip install --upgrade pre-commit + pre-commit install + pre-commit run --all-files + - name: Run unit tests working-directory: xl2times run: | From 04b96fbc9d09e4e0effda7c974bbb12917996fa3 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:53:33 +0200 Subject: [PATCH 15/23] Fix installing from wheel file --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aa4eab..fdd747c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: rm -rf dist || true python -m build # Install the built wheel file to imitiate users installing from PyPI: - python -m pip install --no-index --find-links=dist xl2times + python -m pip install --find-links=dist xl2times - name: Check code formatting working-directory: xl2times From 8e550c3266e486d4e8adec4374aa50a5c0fee3f3 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 09:55:50 +0200 Subject: [PATCH 16/23] Fix --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdd747c..3f55143 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: working-directory: xl2times run: | source .venv/bin/activate + python -m pip install --upgrade pytest pytest # ---------- Prepare ETSAP Demo models From 7577f5a52543b61e45745656422f75a624dd4f12 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:20:28 +0200 Subject: [PATCH 17/23] Install dev dependencies; uninstall xl2times; install from wheel --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f55143..6502369 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,18 +42,20 @@ jobs: run: | python -m venv .venv source .venv/bin/activate - python -m pip install --upgrade pip build + pip install --upgrade pip + pip install -e .[dev] + # Build xl2times rm -rf dist || true python -m build # Install the built wheel file to imitiate users installing from PyPI: - python -m pip install --find-links=dist xl2times + pip uninstall --yes xl2times + pip install --find-links=dist xl2times - name: Check code formatting working-directory: xl2times # Run this step after install so that pyright can find dependencies like pandas run: | source .venv/bin/activate - python -m pip install --upgrade pre-commit pre-commit install pre-commit run --all-files @@ -61,7 +63,6 @@ jobs: working-directory: xl2times run: | source .venv/bin/activate - python -m pip install --upgrade pytest pytest # ---------- Prepare ETSAP Demo models From 7c86f1baac2ba15c2c66001dcd5ec931defda9ad Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:25:19 +0200 Subject: [PATCH 18/23] Don't forget to install build --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6502369..171a797 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: run: | python -m venv .venv source .venv/bin/activate - pip install --upgrade pip + pip install --upgrade pip build pip install -e .[dev] # Build xl2times rm -rf dist || true From 12391dbc8a506a6497eb7d72532ba3a3a110cf3b Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:29:18 +0200 Subject: [PATCH 19/23] Test the publish action --- .github/workflows/ci.yml | 2 +- README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 171a797..425d2d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,7 +185,7 @@ jobs: # ---------- Upload package to PyPI on release - name: Publish to PyPI - if: github.event_name == 'release' && github.event.action == 'published' + # if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ diff --git a/README.md b/README.md index 685dbaa..535eccf 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,6 @@ We recommend installing the tool in editable mode (`-e`) in a Python virtual env python3 -m venv .venv source .venv/bin/activate pip install -U pip -pip install -r requirements.txt pip install -e .[dev] ``` From 0e1ae408f39310de86413d6fa2ec3bdc8aa4ee35 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:38:28 +0200 Subject: [PATCH 20/23] Switch back to publishing only on release --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 425d2d2..171a797 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,7 +185,7 @@ jobs: # ---------- Upload package to PyPI on release - name: Publish to PyPI - # if: github.event_name == 'release' && github.event.action == 'published' + if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ From 9c3a966563e4afd715ab8a51aeb5cdef81fbe9e8 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:39:41 +0200 Subject: [PATCH 21/23] Bump version to be able to publish again on release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a43bb73..21632de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["xl2times", "xl2times.*"] [project] name = "xl2times" -version = "0.1.0-alpha.1" +version = "0.1.0-alpha.2" authors = [ { name="Sam Webster", email="13457618+samwebster@users.noreply.github.com" }, { name="Tom Minka", email="8955276+tminka@users.noreply.github.com" }, From 54d05e80101a841ef1a2cdb5658c858d5c8d686f Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 11:46:05 +0200 Subject: [PATCH 22/23] Release v0.1.0 --- .github/workflows/ci.yml | 1 - pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 171a797..46cb795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,5 +188,4 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 with: - repository-url: https://test.pypi.org/legacy/ packages-dir: xl2times/dist/ diff --git a/pyproject.toml b/pyproject.toml index 21632de..b762ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["xl2times", "xl2times.*"] [project] name = "xl2times" -version = "0.1.0-alpha.2" +version = "0.1.0" authors = [ { name="Sam Webster", email="13457618+samwebster@users.noreply.github.com" }, { name="Tom Minka", email="8955276+tminka@users.noreply.github.com" }, From f1ac068c376c62272c43f0e8ffb0a3e66a9de494 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Sun, 31 Mar 2024 10:54:27 -0400 Subject: [PATCH 23/23] Set version to v0.1.0 (not alpha) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 21632de..b762ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["xl2times", "xl2times.*"] [project] name = "xl2times" -version = "0.1.0-alpha.2" +version = "0.1.0" authors = [ { name="Sam Webster", email="13457618+samwebster@users.noreply.github.com" }, { name="Tom Minka", email="8955276+tminka@users.noreply.github.com" },