From de6d5f161b940ebd052ee3123a7250a0776dcf50 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 7 Feb 2022 15:47:21 +0000 Subject: [PATCH 01/50] Squash all into new skeleton base commit --- .containerignore | 7 + .devcontainer.json | 39 ++++ .gitattributes | 1 - .github/dependabot.yml | 16 ++ .github/pages/index.html | 14 +- .github/workflows/code.yml | 189 ++++++++++++------ .github/workflows/container_tests.sh | 13 ++ .github/workflows/docs.yml | 37 ++-- .github/workflows/docs_clean.yml | 41 ++++ .github/workflows/linkcheck.yml | 34 ++++ .gitignore | 12 +- .gitlab-ci.yml | 4 - .gitremotes | 1 - .pre-commit-config.yaml | 12 +- .vscode/extensions.json | 3 +- .vscode/launch.json | 4 +- .vscode/settings.json | 1 - .vscode/tasks.json | 4 +- CONTRIBUTING.rst | 109 +++++++--- Dockerfile | 47 +++++ Pipfile | 17 -- README.rst | 47 ++--- docs/conf.py | 23 ++- docs/explanations.rst | 2 +- docs/explanations/decisions.rst | 17 ++ .../0001-record-architecture-decisions.rst | 26 +++ docs/explanations/why-is-something-so.rst | 7 - docs/how-to.rst | 2 +- docs/how-to/accomplish-a-task.rst | 7 - docs/{reference => how-to}/contributing.rst | 0 docs/images/dls-logo.svg | 2 +- docs/reference.rst | 5 +- docs/reference/api.rst | 20 +- docs/tutorials/installation.rst | 25 +-- pyproject.toml | 8 +- setup.cfg | 72 +++++-- setup.py | 13 -- src/dls_python3_skeleton/__init__.py | 6 - src/dls_python3_skeleton/__main__.py | 20 -- src/dls_python3_skeleton/_version_git.py | 100 --------- src/dls_python3_skeleton/hello.py | 41 ---- src/python3_pip_skeleton/__init__.py | 12 ++ src/python3_pip_skeleton/__main__.py | 16 ++ tests/test_boilerplate_removed.py | 34 +--- tests/test_cli.py | 9 + tests/test_dls_python3_skeleton.py | 28 --- 46 files changed, 667 insertions(+), 480 deletions(-) create mode 100644 .containerignore create mode 100644 .devcontainer.json delete mode 100644 .gitattributes create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/container_tests.sh create mode 100644 .github/workflows/docs_clean.yml create mode 100644 .github/workflows/linkcheck.yml delete mode 100644 .gitlab-ci.yml delete mode 100644 .gitremotes create mode 100644 Dockerfile delete mode 100644 Pipfile create mode 100644 docs/explanations/decisions.rst create mode 100644 docs/explanations/decisions/0001-record-architecture-decisions.rst delete mode 100644 docs/explanations/why-is-something-so.rst delete mode 100644 docs/how-to/accomplish-a-task.rst rename docs/{reference => how-to}/contributing.rst (100%) delete mode 100644 setup.py delete mode 100644 src/dls_python3_skeleton/__init__.py delete mode 100644 src/dls_python3_skeleton/__main__.py delete mode 100644 src/dls_python3_skeleton/_version_git.py delete mode 100644 src/dls_python3_skeleton/hello.py create mode 100644 src/python3_pip_skeleton/__init__.py create mode 100644 src/python3_pip_skeleton/__main__.py create mode 100644 tests/test_cli.py delete mode 100644 tests/test_dls_python3_skeleton.py diff --git a/.containerignore b/.containerignore new file mode 100644 index 00000000..eb7d5ae1 --- /dev/null +++ b/.containerignore @@ -0,0 +1,7 @@ +Dockerfile +build/ +dist/ +.mypy_cache +.tox +.venv* +venv* diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 00000000..d0921df6 --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,39 @@ +// For format details, see https://aka.ms/devcontainer.json +{ + "name": "Python 3 Developer Container", + "build": { + "dockerfile": "Dockerfile", + "target": "build", + "context": ".", + "args": {} + }, + "remoteEnv": { + "DISPLAY": "${localEnv:DISPLAY}" + }, + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/venv/bin/python", + "python.linting.enabled": true + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + // Make sure the files we are mapping into the container exist on the host + "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'", + "runArgs": [ + "--net=host", + "-v=${localEnv:HOME}/.ssh:/root/.ssh", + "-v=${localEnv:HOME}/.inputrc:/root/.inputrc" + ], + "mounts": [ + // map in home directory - not strictly necessary but useful + "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached" + ], + // make the workspace folder the same inside and outside of the container + "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind", + "workspaceFolder": "${localWorkspaceFolder}", + // After the container is created, install the python project in editable form + "postCreateCommand": "pip install $([ -f requirements_dev.txt ] && echo -r requirements_dev.txt ) -e .[dev]" +} \ No newline at end of file diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 075748c4..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -src/*/_version_git.py export-subst diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..fb7c6ee6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/pages/index.html b/.github/pages/index.html index cc33127d..80f0a009 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -1,9 +1,11 @@ - - Redirecting to master branch + + + Redirecting to main branch - - - - + + + + + \ No newline at end of file diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 62b51616..5ea6e1f2 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -2,67 +2,39 @@ name: Code CI on: push: - branches: - # Restricting to these branches and tags stops duplicate jobs on internal - # PRs but stops CI running on internal branches without a PR. Delete the - # next 5 lines to restore the original behaviour - - master - - main - tags: - - "*" pull_request: schedule: # Run every Monday at 8am to check latest versions of dependencies - - cron: '0 8 * * MON' + - cron: "0 8 * * WED" jobs: lint: - runs-on: "ubuntu-latest" - steps: - - name: Run black, flake8, mypy - uses: dls-controls/pipenv-run-action@v1 - with: - pipenv-run: lint + # pull requests are a duplicate of a branch push if within the same repo. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest - wheel: - runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup python + uses: actions/setup-python@v4 with: - fetch-depth: 0 + python-version: "3.10" - - name: Create Sdist and Wheel - # Set SOURCE_DATE_EPOCH from git commit for reproducible build - # https://reproducible-builds.org/ - # Set group writable and umask to do the same to match inside DLS + - name: Lint run: | - chmod -R g+w . - umask 0002 - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) pipx run build --sdist --wheel - - - name: Test cli works from the installed wheel - # Can remove the repository reference after https://github.com/pypa/pipx/pull/733 - run: pipx run --spec dist/*.whl ${GITHUB_REPOSITORY##*/} --version - - - name: Upload Wheel and Sdist as artifacts - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/* + touch requirements_dev.txt requirements.txt + pip install -r requirements.txt -r requirements_dev.txt -e .[dev] + tox -e pre-commit,mypy test: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.7", "3.8", "3.9"] - pipenv: ["skip-lock"] - - include: - # Add an extra Python3.7 runner to use the lockfile - - os: "ubuntu-latest" - python: "3.7" - pipenv: "deploy" + os: ["ubuntu-latest"] # can add windows-latest, macos-latest + python: ["3.8", "3.9", "3.10"] runs-on: ${{ matrix.os }} env: @@ -70,37 +42,130 @@ jobs: PY_IGNORE_IMPORTMISMATCH: "1" steps: - - name: Setup repo and test - uses: dls-controls/pipenv-run-action@v1 + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup python ${{ matrix.python }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - pipenv-install: --dev --${{ matrix.pipenv }} - allow-editable-installs: ${{ matrix.pipenv == 'deploy' }} - pipenv-run: tests + + - name: Install with latest dependencies + run: pip install .[dev] + + - name: Run tests + run: pytest tests - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: - name: ${{ matrix.python }}/${{ matrix.os }}/${{ matrix.pipenv }} + name: ${{ matrix.python }}/${{ matrix.os }} files: cov.xml - release: - needs: [lint, wheel, test] + container: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest - # upload to PyPI and make a release on every tag - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + permissions: + contents: read + packages: write + steps: - - uses: actions/download-artifact@v2 + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=tag + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build developer image for testing + uses: docker/build-push-action@v3 + with: + tags: build:latest + context: . + target: build + load: true + + - name: Run tests in the container locked with requirements_dev.txt + run: | + docker run --name test build bash /project/.github/workflows/container_tests.sh + docker cp test:/project/dist . + docker cp test:/project/cov.xml . + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + name: 3.10-locked/ubuntu-latest + files: cov.xml + + - name: Build runtime image + uses: docker/build-push-action@v3 + with: + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + context: . + labels: ${{ steps.meta.outputs.labels }} + + - name: Check runtime + run: for i in ${{ steps.meta.outputs.tags }}; do docker run ${i} --version; done + + - name: Upload build files + uses: actions/upload-artifact@v3 with: name: dist - path: dist + path: dist/* + + sdist: + needs: container + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v3 + + - name: Install sdist in a venv and check cli works + # ${GITHUB_REPOSITORY##*/} is the repo name without org + # Replace this with the cli command if different to the repo name + run: | + pip install dist/*.gz + ${GITHUB_REPOSITORY##*/} --version + + release: + # upload to PyPI and make a release on every tag + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + needs: container + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v3 - name: Github Release # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14 + uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14 with: - files: dist/* + prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} + files: | + dist/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -108,5 +173,5 @@ jobs: - name: Publish to PyPI env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.pypi_token }} - run: pipx run twine upload dist/* + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: pipx run twine upload dist/*/whl dist/*.tar.gz diff --git a/.github/workflows/container_tests.sh b/.github/workflows/container_tests.sh new file mode 100644 index 00000000..5f921597 --- /dev/null +++ b/.github/workflows/container_tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -x + +cd /project +source /venv/bin/activate + +touch requirements_dev.txt +pip install -r requirements_dev.txt -e .[dev] +pip freeze --exclude-editable > dist/requirements_dev.txt + +pipdeptree + +pytest tests diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e3b8fc90..c3c5eda8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,16 +2,16 @@ name: Docs CI on: push: - branches: - # Add more branches here to publish docs from other branches - - master - - main - tags: - - "*" pull_request: jobs: docs: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + matrix: + python: ["3.10"] + runs-on: ubuntu-latest steps: @@ -19,27 +19,40 @@ jobs: if: startsWith(github.ref, 'refs/tags') run: sleep 60 + - name: Install python version + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install Packages # Can delete this if you don't use graphviz in your docs run: sudo apt-get install graphviz - - name: Build docs - uses: dls-controls/pipenv-run-action@v1 + - name: checkout + uses: actions/checkout@v2 with: - pipenv-run: docs + fetch-depth: 0 + + - name: Install dependencies + run: | + touch requirements_dev.txt + pip install -r requirements_dev.txt -e .[dev] + + - name: Build docs + run: tox -e docs - name: Move to versioned directory - # e.g. master or 0.1.2 + # e.g. main or 0.1.2 run: mv build/html ".github/pages/${GITHUB_REF##*/}" - name: Write versions.txt - run: pipenv run sphinx_rtd_theme_github_versions .github/pages + run: sphinx_rtd_theme_github_versions .github/pages - name: Publish Docs to gh-pages if: github.event_name == 'push' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 + uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml new file mode 100644 index 00000000..2059c7f3 --- /dev/null +++ b/.github/workflows/docs_clean.yml @@ -0,0 +1,41 @@ +name: Docs Cleanup CI + +# delete branch documentation when a branch is deleted +# also allow manually deleting a documentation version +on: + delete: + workflow_dispatch: + inputs: + version: + description: "documentation version to DELETE" + required: true + type: string + +jobs: + remove: + if: github.event.ref_type == 'branch' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + ref: gh-pages + + - name: removing documentation for branch ${{ github.event.ref }} + if: ${{ github.event_name != 'workflow_dispatch' }} + run: echo "remove_me=${{ github.event.ref }}" >> $GITHUB_ENV + + - name: manually removing documentation version ${{ github.event.inputs.version }} + if: ${{ github.event_name == 'workflow_dispatch' }} + run: echo "remove_me=${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: update index and push changes + run: | + echo removing redundant documentation version ${{ env.remove_me }} + rm -r ${{ env.remove_me }} + sed -i /${{ env.remove_me }}/d versions.txt + git config --global user.name 'GitHub Actions Docs Cleanup CI' + git config --global user.email 'gha@users.noreply.github.com' + git commit -am"removing redundant docs version ${{ env.remove_me }}" + git push diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml new file mode 100644 index 00000000..6b37f194 --- /dev/null +++ b/.github/workflows/linkcheck.yml @@ -0,0 +1,34 @@ +name: Link Check + +on: + schedule: + # Run every Monday at 8am to check URL links still resolve + - cron: "0 8 * * WED" + +jobs: + docs: + strategy: + fail-fast: false + matrix: + python: ["3.10"] + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install python version + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies + run: | + touch requirements_dev.txt + pip install -r requirements_dev.txt -e .[dev] + + - name: Check links + run: tox -e docs -- -b linkcheck diff --git a/.gitignore b/.gitignore index 0ce69d99..e0fba46a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv build/ develop-eggs/ dist/ @@ -22,6 +23,7 @@ var/ *.egg-info/ .installed.cfg *.egg +**/_version.py # PyInstaller # Usually these files are written by a python script from a template @@ -58,8 +60,8 @@ docs/_build/ # PyBuilder target/ -# DLS build dir and virtual environment -/prefix/ -/venv/ -/lightweight-venv/ -/installed.files +# likely venv names +.venv* +venv* + + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 1efd5024..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,4 +0,0 @@ -include: - - project: 'controls/reports/ci_templates' - ref: master - file: 'python3/dls_py3_template.yml' diff --git a/.gitremotes b/.gitremotes deleted file mode 100644 index 3d3f0349..00000000 --- a/.gitremotes +++ /dev/null @@ -1 +0,0 @@ -github git@github.com:dls-controls/dls-python3-skeleton.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23a81d4e..5e270b08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,20 +12,12 @@ repos: name: Run black stages: [commit] language: system - entry: pipenv run black --check --diff + entry: black --check --diff types: [python] - id: flake8 name: Run flake8 stages: [commit] language: system - entry: pipenv run flake8 + entry: flake8 types: [python] - exclude: setup.py - - - id: mypy - name: Run mypy - stages: [commit] - language: system - entry: pipenv run mypy src tests - pass_filenames: false diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 734f215e..d173f8d6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,7 +1,8 @@ { "recommendations": [ + "ms-vscode-remote.remote-containers" "ms-python.vscode-pylance", "ms-python.python", "ryanluker.vscode-coverage-gutters" ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index 1d960dc9..f8fcdb4f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,9 @@ "request": "launch", "justMyCode": false, "program": "${file}", - "purpose": ["debug-test"], + "purpose": [ + "debug-test" + ], "console": "integratedTerminal", "env": { // The default config in setup.cfg's "[tool:pytest]" adds coverage. diff --git a/.vscode/settings.json b/.vscode/settings.json index 192c474e..2472acfd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "editor.defaultFormatter": "ms-python.python", "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": true, "python.linting.mypyEnabled": true, diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ff78a11b..946e69d4 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -5,8 +5,8 @@ "tasks": [ { "type": "shell", - "label": "Tests with coverage", - "command": "pipenv run tests", + "label": "Tests, lint and docs", + "command": "tox -p", "options": { "cwd": "${workspaceRoot}" }, diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index eba66b59..327e3920 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,17 +7,45 @@ filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project. -.. _GitHub: https://github.com/dls-controls/dls-python3-skeleton/issues +.. _GitHub: https://github.com/epics-containers/python3-pip-skeleton/issues Running the tests ----------------- -To get the source source code and run the unit tests, run:: +To run in a container +~~~~~~~~~~~~~~~~~~~~~ - $ git clone git://github.com/dls-controls/dls-python3-skeleton.git - $ cd dls-python3-skeleton - $ pipenv install --dev - $ pipenv run tests +Use vscode devcontainer as follows:: + + $ git clone git://github.com/epics-containers/python3-pip-skeleton.git + $ vscode python3-pip-skeleton + Click on 'Reopen in Container' when prompted + In a vscode Terminal: + $ tox -p + + +To run locally +~~~~~~~~~~~~~~ + +Get the source source code and run the unit tests directly +on your workstation as follows:: + + $ git clone git://github.com/epics-containers/python3-pip-skeleton.git + $ cd python3-pip-skeleton + $ virtualenv .venv + $ source .venv/bin/activate + $ pip install -e .[dev] + $ tox -p + +In both cases tox -p runs in parallel the following checks: + + - Build Sphinx Documentation + - run pytest on all tests in ./tests + - run mypy linting on all files in ./src ./tests + - run pre-commit checks: + + - run flake8 style checks against all source + - run black formatting checks against all source While 100% code coverage does not make a library bug-free, it significantly reduces the number of easily caught bugs! Please make sure coverage remains the @@ -33,22 +61,43 @@ The code in this repository conforms to standards set by the following tools: - isort_ for import ordering - mypy_ for static type checking -These checks will be run by pre-commit_. You can either choose to run these -tests on all files tracked by git:: +flake8 and black and isort are run by pre-commit_. You can run the above checks on +all files with this command:: - $ pipenv run lint + $ tox -e pre-commit,mypy Or you can install a pre-commit hook that will run each time you do a ``git -commit`` on just the files that have changed:: +commit`` on just the files that have changed. Note that mypy is not in +the pre-commit because it is a little slow :: - $ pipenv run pre-commit install + $ pre-commit install .. _black: https://github.com/psf/black -.. _flake8: http://flake8.pycqa.org/en/latest/ -.. _isort: https://github.com/timothycrosley/isort +.. _flake8: https://flake8.pycqa.org/en/latest/ +.. _isort: https://github.com/PyCQA/isort .. _mypy: https://github.com/python/mypy .. _pre-commit: https://pre-commit.com/ +Docstrings are pre-processed using the Sphinx Napoleon extension. As such, +google-style_ is considered as standard for this repository. Please use type +hints in the function signature for types. For example:: + + def func(arg1: str, arg2: int) -> bool: + """Summary line. + + Extended description of function. + + Args: + arg1: Description of arg1 + arg2: Description of arg2 + + Returns: + Description of return value + """ + return True + +.. _google-style: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/index.html#google-vs-numpy + Documentation ------------- @@ -68,7 +117,7 @@ Docs follow the underlining convention:: You can build the docs from the project directory by running:: - $ pipenv run docs + $ tox -e docs $ firefox build/html/index.html Release Process @@ -76,27 +125,39 @@ Release Process To make a new release, please follow this checklist: -- Choose a new PEP440 compliant release number -- Git tag the version -- Push to GitHub and the actions will make a release on pypi -- Push to internal gitlab and do a dls-release.py of the tag -- Check and edit for clarity the autogenerated GitHub release_ +- Choose a new PEP440 compliant release number (see https://peps.python.org/pep-0440/) +- Go to the GitHub release_ page +- Choose ``Draft New Release`` +- Click ``Choose Tag`` and supply the new tag you chose (click create new tag) +- Click ``Generate release notes``, review and edit these notes +- Choose a title and click ``Publish Release`` + +Note that tagging and pushing to the main branch has the same effect except that +you will not get the option to edit the release notes. + +.. _release: https://github.com/epics-containers/python3-pip-skeleton/releases + + +Checking Dependencies +--------------------- + +To see a graph of the python package dependency tree type:: -.. _release: https://dls-controls.github.io/dls-python3-skeleton/releases + pipdeptree Updating the tools ------------------ -This module is merged with the dls-python3-skeleton_. This is a generic +This module is merged with the python3-pip-skeleton_. This is a generic Python project structure which provides a means to keep tools and techniques in sync between multiple Python projects. To update to the latest version of the skeleton, run:: - $ git pull https://github.com/dls-controls/dls-python3-skeleton skeleton + $ git pull https://github.com/dls-controls/python3-pip-skeleton main Any merge conflicts will indicate an area where something has changed that conflicts with the setup of the current module. Check the `closed pull requests -`_ +`_ of the skeleton module for more details. -.. _dls-python3-skeleton: https://dls-controls.github.io/dls-python3-skeleton +.. _python3-pip-skeleton: https://epics-containers.github.io/python3-pip-skeleton diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..76b5dad3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# This file is for use as a devcontainer and a runtime container +# +# The devcontainer should use the build target and run as root with podman +# or docker with user namespaces. +# +FROM python:3.10 as build + +# Add any system dependencies for the developer/build environment here +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + busybox \ + git \ + net-tools \ + vim \ + && rm -rf /var/lib/apt/lists/* \ + && busybox --install + +COPY . /project + +RUN cd /project && \ + pip install --upgrade pip build && \ + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + python -m build --sdist --wheel && \ + touch requirements.txt + +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH + +RUN cd /project && \ + pip install --upgrade pip && \ + pip install -r requirements.txt dist/*.whl && \ + pip freeze > dist/requirements.txt && \ + # we don't want to include our own wheel in requirements - remove with sed + # and replace with a comment to avoid a zero length asset upload later + sed -i '/file:/s/^/# Requirements for /' dist/requirements.txt + +FROM python:3.10-slim as runtime + +# Add apt-get system dependecies for runtime here if needed + +COPY --from=build /venv/ /venv/ +ENV PATH=/venv/bin:$PATH + +# change this entrypoint if it is not the same as the repo +ENTRYPOINT ["python3-pip-skeleton"] +CMD ["--version"] diff --git a/Pipfile b/Pipfile deleted file mode 100644 index d1ebb0d9..00000000 --- a/Pipfile +++ /dev/null @@ -1,17 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] -dls_python3_skeleton = {editable = true, extras = ["dev"], path = "."} - -[packages] -dls_python3_skeleton = {editable = true, path = "."} - -[scripts] -lint = "pre-commit run --all-files --show-diff-on-failure --color=always -v" -tests = "pytest" -docs = "sphinx-build -EWT --keep-going docs build/html" -# Delete any files that git ignore hides from us -gitclean = "git clean -fdX" diff --git a/README.rst b/README.rst index 9b99e2cb..360d0ef6 100644 --- a/README.rst +++ b/README.rst @@ -1,47 +1,44 @@ -dls-python3-skeleton +python3-pip-skeleton =========================== |code_ci| |docs_ci| |coverage| |pypi_version| |license| +.. note:: + + This project contains template code only. For documentation on how to + adopt this skeleton project see + https://epics-containers.github.io/python3-pip-skeleton-cli + This is where you should write a short paragraph that describes what your module does, how it does it, and why people should use it. ============== ============================================================== -PyPI ``pip install dls_python3_skeleton`` -Source code https://github.com/dls-controls/dls-python3-skeleton -Documentation https://dls-controls.github.io/dls-python3-skeleton -Releases https://github.com/dls-controls/dls-python3-skeleton/releases +PyPI ``pip install python3-pip-skeleton`` +Source code https://github.com/epics-containers/python3-pip-skeleton +Documentation https://epics-containers.github.io/python3-pip-skeleton +Releases https://github.com/epics-containers/python3-pip-skeleton/releases ============== ============================================================== This is where you should put some images or code snippets that illustrate some relevant examples. If it is a library then you might put some -introductory code here: - -.. code:: python - - from dls_python3_skeleton.hello import HelloClass - - hello = HelloClass("me") - print(hello.format_greeting()) - -Or if it is a commandline tool then you might put some example commands here:: +introductory code here. - dls-python3-skeleton person --times=2 +Or if it is a commandline tool then you might put some example commands here. -.. |code_ci| image:: https://github.com/dls-controls/dls-python3-skeleton/workflows/Code%20CI/badge.svg?branch=master - :target: https://github.com/dls-controls/dls-python3-skeleton/actions?query=workflow%3A%22Code+CI%22 +.. |code_ci| image:: https://github.com/epics-containers/python3-pip-skeleton/workflows/Code%20CI/badge.svg?branch=main + :target: https://github.com/epics-containers/python3-pip-skeleton/actions?query=workflow%3A%22Code+CI%22 :alt: Code CI -.. |docs_ci| image:: https://github.com/dls-controls/dls-python3-skeleton/workflows/Docs%20CI/badge.svg?branch=master - :target: https://github.com/dls-controls/dls-python3-skeleton/actions?query=workflow%3A%22Docs+CI%22 +.. |docs_ci| image:: https://github.com/epics-containers/python3-pip-skeleton/workflows/Docs%20CI/badge.svg?branch=main + :target: https://github.com/epics-containers/python3-pip-skeleton/actions?query=workflow%3A%22Docs+CI%22 :alt: Docs CI -.. |coverage| image:: https://codecov.io/gh/dls-controls/dls-python3-skeleton/branch/master/graph/badge.svg - :target: https://codecov.io/gh/dls-controls/dls-python3-skeleton +.. |coverage| image:: https://codecov.io/gh/epics-containers/python3-pip-skeleton/branch/main/graph/badge.svg + :target: https://codecov.io/gh/epics-containers/python3-pip-skeleton :alt: Test Coverage -.. |pypi_version| image:: https://img.shields.io/pypi/v/dls_python3_skeleton.svg - :target: https://pypi.org/project/dls_python3_skeleton +.. |pypi_version| image:: https://img.shields.io/pypi/v/python3-pip-skeleton.svg + :target: https://pypi.org/project/python3-pip-skeleton :alt: Latest PyPI version .. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg @@ -52,4 +49,4 @@ Or if it is a commandline tool then you might put some example commands here:: Anything below this line is used when viewing README.rst and will be replaced when included in index.rst -See https://dls-controls.github.io/dls-python3-skeleton for more detailed documentation. +See https://epics-containers.github.io/python3-pip-skeleton for more detailed documentation. diff --git a/docs/conf.py b/docs/conf.py index f2ca6a2d..bee41d4b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,20 +4,20 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -import dls_python3_skeleton +import python3_pip_skeleton # -- General configuration ------------------------------------------------ # General information about the project. -project = "dls-python3-skeleton" +project = "python3-pip-skeleton" # The full version, including alpha/beta/rc tags. -release = dls_python3_skeleton.__version__ +release = python3_pip_skeleton.__version__ # The short X.Y version. if "+" in release: # Not on a tag - version = "master" + version = "main" else: version = release @@ -42,7 +42,17 @@ # generating warnings in "nitpicky mode". Note that type should include the # domain name if present. Example entries would be ('py:func', 'int') or # ('envvar', 'LD_LIBRARY_PATH'). -nitpick_ignore = [("py:func", "int")] +nitpick_ignore = [ + ("py:class", "NoneType"), + ("py:class", "'str'"), + ("py:class", "'float'"), + ("py:class", "'int'"), + ("py:class", "'bool'"), + ("py:class", "'object'"), + ("py:class", "'id'"), + ("py:class", "apischema.utils.UndefinedType"), + ("py:class", "typing_extensions.Literal"), +] # Both the class’ and the __init__ method’s docstring are concatenated and # inserted into the main body of the autoclass directive @@ -88,6 +98,9 @@ http://www.diamond.ac.uk """ +# Ignore localhost links for period check that links in docs are valid +linkcheck_ignore = [r"http://localhost:\d+/"] + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/explanations.rst b/docs/explanations.rst index 39de4e6b..1e329673 100644 --- a/docs/explanations.rst +++ b/docs/explanations.rst @@ -8,4 +8,4 @@ Explanation of how the library works and why it works that way. .. toctree:: :caption: Explanations - explanations/why-is-something-so + explanations/decisions diff --git a/docs/explanations/decisions.rst b/docs/explanations/decisions.rst new file mode 100644 index 00000000..5841e6ea --- /dev/null +++ b/docs/explanations/decisions.rst @@ -0,0 +1,17 @@ +.. This Source Code Form is subject to the terms of the Mozilla Public +.. License, v. 2.0. If a copy of the MPL was not distributed with this +.. file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Architectural Decision Records +============================== + +We record major architectural decisions in Architecture Decision Records (ADRs), +as `described by Michael Nygard +`_. +Below is the list of our current ADRs. + +.. toctree:: + :maxdepth: 1 + :glob: + + decisions/* \ No newline at end of file diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.rst b/docs/explanations/decisions/0001-record-architecture-decisions.rst new file mode 100644 index 00000000..b2d3d0fe --- /dev/null +++ b/docs/explanations/decisions/0001-record-architecture-decisions.rst @@ -0,0 +1,26 @@ +1. Record architecture decisions +================================ + +Date: 2022-02-18 + +Status +------ + +Accepted + +Context +------- + +We need to record the architectural decisions made on this project. + +Decision +-------- + +We will use Architecture Decision Records, as `described by Michael Nygard +`_. + +Consequences +------------ + +See Michael Nygard's article, linked above. To create new ADRs we will copy and +paste from existing ones. diff --git a/docs/explanations/why-is-something-so.rst b/docs/explanations/why-is-something-so.rst deleted file mode 100644 index 21708377..00000000 --- a/docs/explanations/why-is-something-so.rst +++ /dev/null @@ -1,7 +0,0 @@ -Why is something the way it is -============================== - -Often, reading the code will not explain *why* it is written that way. These -explanations should be grouped together in articles here. They might include -history of dls-python3-skeleton, architectural decisions, or the -real world tests that influenced the design of dls-python3-skeleton. diff --git a/docs/how-to.rst b/docs/how-to.rst index 86b2ddbd..700797cc 100644 --- a/docs/how-to.rst +++ b/docs/how-to.rst @@ -8,4 +8,4 @@ Practical step-by-step guides for the more experienced user. .. toctree:: :caption: How-to Guides - how-to/accomplish-a-task + how-to/contributing diff --git a/docs/how-to/accomplish-a-task.rst b/docs/how-to/accomplish-a-task.rst deleted file mode 100644 index 8ee49390..00000000 --- a/docs/how-to/accomplish-a-task.rst +++ /dev/null @@ -1,7 +0,0 @@ -How to accomplish a task -======================== - -Here you would explain how to use dls-python3-skeleton to accomplish -a particular task. It doesn't have to be an exhaustive guide like the tutorials, -just enough information to show someone who knows what they want to do, how to -accomplish that task. diff --git a/docs/reference/contributing.rst b/docs/how-to/contributing.rst similarity index 100% rename from docs/reference/contributing.rst rename to docs/how-to/contributing.rst diff --git a/docs/images/dls-logo.svg b/docs/images/dls-logo.svg index 79ba266f..0af1a177 100644 --- a/docs/images/dls-logo.svg +++ b/docs/images/dls-logo.svg @@ -8,4 +8,4 @@ - + \ No newline at end of file diff --git a/docs/reference.rst b/docs/reference.rst index 3f01ee35..bfa7a4f4 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -3,14 +3,13 @@ Reference ========= -Practical step-by-step guides for the more experienced user. +Technical reference material including APIs and release notes. .. toctree:: :caption: Reference reference/api - reference/contributing - Releases + Releases Index .. diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 06bf3c66..8544e172 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -1,24 +1,14 @@ API === -.. automodule:: dls_python3_skeleton +.. automodule:: python3_pip_skeleton - ``dls_python3_skeleton`` + ``python3_pip_skeleton`` ----------------------------------- -This is the internal API reference for dls_python3_skeleton +This is the internal API reference for python3_pip_skeleton -You can mix verbose text with docstring and signature extraction by -using ``autoclass`` and ``autofunction`` directives instead of -``automodule`` below. - -.. data:: dls_python3_skeleton.__version__ +.. data:: python3_pip_skeleton.__version__ :type: str - Version number as calculated by https://github.com/dls-controls/versiongit - -.. automodule:: dls_python3_skeleton.hello - :members: - - ``dls_python3_skeleton.hello`` - ----------------------------------------- + Version number as calculated by https://github.com/pypa/setuptools_scm diff --git a/docs/tutorials/installation.rst b/docs/tutorials/installation.rst index d23580a3..bbffd015 100644 --- a/docs/tutorials/installation.rst +++ b/docs/tutorials/installation.rst @@ -1,20 +1,10 @@ Installation ============ -.. note:: - - For installation inside DLS, please see the internal documentation on - ``dls-python3`` and ``pipenv``. Although these instructions will work - inside DLS, they are intended for external use. - - If you want to contribute to the library itself, please follow - the `../reference/contributing` instructions. - - Check your version of python ---------------------------- -You will need python 3.7 or later. You can check your version of python by +You will need python 3.8 or later. You can check your version of python by typing into a terminal:: python3 --version @@ -35,14 +25,21 @@ Installing the library You can now use ``pip`` to install the library:: - python3 -m pip install dls_python3_skeleton + python3 -m pip install python3-pip-skeleton If you require a feature that is not currently released you can also install from github:: - python3 -m pip install git+git://github.com/dls-controls/dls-python3-skeleton.git + python3 -m pip install git+https://github.com/epics-containers/python3-pip-skeleton.git The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing:: - dls-python3-skeleton --version + python3-pip-skeleton --version + +Running in a container +---------------------- + +To pull the container from github container registry and run:: + + docker run ghcr.io/epics-containers/python3-pip-skeleton:main --version diff --git a/pyproject.toml b/pyproject.toml index ccd70c76..1b8c998a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -# To get a reproducible wheel, wheel must be pinned to the same version as in -# dls-python3, and setuptools must produce the same dist-info. Cap setuptools -# to the last version that didn't add License-File to METADATA -requires = ["setuptools<57", "wheel==0.33.1"] +requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel"] build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/python3_pip_skeleton/_version.py" diff --git a/setup.cfg b/setup.cfg index 167860a5..008fd8a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,24 +1,29 @@ [metadata] -name = dls_python3_skeleton +name = python3-pip-skeleton description = One line description of your module -url = https://github.com/dls-controls/dls-python3-skeleton +url = https://github.com/epics-containers/python3-pip-skeleton author = Firstname Lastname author_email = email@address.com license = Apache License 2.0 long_description = file: README.rst long_description_content_type = text/x-rst classifiers = - Development Status :: 4 - Beta License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [options] -python_requires = >=3.7 +python_requires = >=3.8 packages = find: +# =src is interpreted as {"": "src"} +# as per recommendation here https://hynek.me/articles/testing-packaging/ package_dir = =src + +setup_requires = + setuptools_scm[toml]>=6.2 + # Specify any package dependencies below. # install_requires = # numpy @@ -27,26 +32,32 @@ package_dir = [options.extras_require] # For development tests/docs dev = - black==21.9b0 + black==22.6.0 + flake8-isort isort>5.0 - pytest-cov mypy - flake8-isort - sphinx-rtd-theme-github-versions + pipdeptree pre-commit + pytest-cov + setuptools_scm[toml]>=6.2 + sphinx-rtd-theme-github-versions + tox + types-mock [options.packages.find] where = src +# Don't include our tests directory in the distribution +exclude = tests # Specify any package data to be included in the wheel below. # [options.package_data] -# dls_python3_skeleton = +# python3_pip_skeleton = # subpackage/*.yaml [options.entry_points] # Include a command line script console_scripts = - dls-python3-skeleton = dls_python3_skeleton.__main__:main + python3-pip-skeleton = python3_pip_skeleton.__main__:main [mypy] # Ignore missing stubs for modules we use @@ -55,7 +66,6 @@ ignore_missing_imports = True [isort] profile=black float_to_top=true -skip=setup.py,conf.py,build [flake8] # Make flake8 respect black's line length (default 88), @@ -63,22 +73,52 @@ max-line-length = 88 extend-ignore = E203, # See https://github.com/PyCQA/pycodestyle/issues/373 F811, # support typing.overload decorator + F722, # allow Annotated[typ, some_func("some string")] +exclude = + ui_* + .tox + .venv [tool:pytest] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = --tb=native -vv --doctest-modules --doctest-glob="*.rst" - --cov=dls_python3_skeleton --cov-report term --cov-report xml:cov.xml + --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings filterwarnings = error +# Doctest python code in docs, python code in src docstrings, test functions in tests +testpaths = + docs src tests [coverage:run] -# This is covered in the versiongit test suite so exclude it here -omit = */_version_git.py -data_file = /tmp/dls_python3_skeleton.coverage +data_file = /tmp/python3_pip_skeleton.coverage [coverage:paths] # Tests are run from installed location, map back to the src directory source = src **/site-packages/ + +# Use tox to provide parallel linting and testing +# NOTE that we pre-install all tools in the dev dependencies (including tox). +# Hence the use of allowlist_externals instead of using the tox virtualenvs. +# This ensures a match between developer time tools in the IDE and tox tools. +[tox:tox] +minversion = 3.7 +skipsdist=true +skipinstall=true + +[testenv:{pre-commit,mypy,pytest,docs}] +passenv = + PYTHONPATH + HOME +allowlist_externals = + pytest + pre-commit + mypy + sphinx-build +commands = + pytest: pytest tests {posargs} + mypy: mypy src tests {posargs} + pre-commit: pre-commit run --all-files {posargs} + docs: sphinx-build -EWT --keep-going docs build/html {posargs} diff --git a/setup.py b/setup.py deleted file mode 100644 index 39de827e..00000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -# type: ignore -import glob -import importlib.util - -from setuptools import setup - -# Import ._version_git.py without importing -path = glob.glob(__file__.replace("setup.py", "src/*/_version_git.py"))[0] -spec = importlib.util.spec_from_file_location("_version_git", path) -vg = importlib.util.module_from_spec(spec) -spec.loader.exec_module(vg) - -setup(cmdclass=vg.get_cmdclass(), version=vg.__version__) diff --git a/src/dls_python3_skeleton/__init__.py b/src/dls_python3_skeleton/__init__.py deleted file mode 100644 index 25eb98fa..00000000 --- a/src/dls_python3_skeleton/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import hello -from ._version_git import __version__ - -# __all__ defines the public API for the package. -# Each module also defines its own __all__. -__all__ = ["__version__", "hello"] diff --git a/src/dls_python3_skeleton/__main__.py b/src/dls_python3_skeleton/__main__.py deleted file mode 100644 index 960eff10..00000000 --- a/src/dls_python3_skeleton/__main__.py +++ /dev/null @@ -1,20 +0,0 @@ -from argparse import ArgumentParser - -from . import __version__ -from .hello import HelloClass, say_hello_lots - -__all__ = ["main"] - - -def main(args=None): - parser = ArgumentParser() - parser.add_argument("--version", action="version", version=__version__) - parser.add_argument("name", help="Name of the person to greet") - parser.add_argument("--times", type=int, default=5, help="Number of times to greet") - args = parser.parse_args(args) - say_hello_lots(HelloClass(args.name), args.times) - - -# test with: pipenv run python -m dls_python3_skeleton -if __name__ == "__main__": - main() diff --git a/src/dls_python3_skeleton/_version_git.py b/src/dls_python3_skeleton/_version_git.py deleted file mode 100644 index bb7f0c2a..00000000 --- a/src/dls_python3_skeleton/_version_git.py +++ /dev/null @@ -1,100 +0,0 @@ -# Compute a version number from a git repo or archive - -# This file is released into the public domain. Generated by: -# versiongit-2.1 (https://github.com/dls-controls/versiongit) -import re -import sys -from pathlib import Path -from subprocess import STDOUT, CalledProcessError, check_output - -# These will be filled in if git archive is run or by setup.py cmdclasses -GIT_REFS = "$Format:%D$" -GIT_SHA1 = "$Format:%h$" - -# Git describe gives us sha1, last version-like tag, and commits since then -CMD = "git describe --tags --dirty --always --long --match=[0-9]*[-.][0-9]*" - - -def get_version_from_git(path=None): - """Try to parse version from git describe, fallback to git archive tags.""" - tag, plus, suffix = "0.0", "untagged", "" - if not GIT_SHA1.startswith("$"): - # git archive or the cmdclasses below have filled in these strings - sha1 = GIT_SHA1 - for ref_name in GIT_REFS.split(", "): - if ref_name.startswith("tag: "): - # git from 1.8.3 onwards labels archive tags "tag: TAGNAME" - tag, plus = ref_name[5:], "0" - else: - if path is None: - # If no path to git repo, choose the directory this file is in - path = Path(__file__).absolute().parent - # output is TAG-NUM-gHEX[-dirty] or HEX[-dirty] - try: - cmd_out = check_output(CMD.split(), stderr=STDOUT, cwd=path) - except Exception as e: - sys.stderr.write("%s: %s\n" % (type(e).__name__, str(e))) - if isinstance(e, CalledProcessError): - sys.stderr.write("-> %s" % e.output.decode()) - return "0.0+unknown", None, e - else: - out = cmd_out.decode().strip() - if out.endswith("-dirty"): - out = out[:-6] - suffix = ".dirty" - if "-" in out: - # There is a tag, extract it and the other pieces - match = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", out) - tag, plus, sha1 = match.groups() - else: - # No tag, just sha1 - sha1 = out - # Replace dashes in tag for dots - tag = tag.replace("-", ".") - if plus != "0" or suffix: - # Not on a tag, add additional info - tag = f"{tag}+{plus}.g{sha1}{suffix}" - return tag, sha1, None - - -__version__, git_sha1, git_error = get_version_from_git() - - -def get_cmdclass(build_py=None, sdist=None): - """Create cmdclass dict to pass to setuptools.setup. - - Create cmdclass dict to pass to setuptools.setup which will write a - _version_static.py file in our resultant sdist, wheel or egg. - """ - if build_py is None: - from setuptools.command.build_py import build_py - if sdist is None: - from setuptools.command.sdist import sdist - - def make_version_static(base_dir: str, pkg: str): - vg = Path(base_dir) / pkg.split(".")[0] / "_version_git.py" - if vg.is_file(): - lines = open(vg).readlines() - with open(vg, "w") as f: - for line in lines: - # Replace GIT_* with static versions - if line.startswith("GIT_SHA1 = "): - f.write("GIT_SHA1 = '%s'\n" % git_sha1) - elif line.startswith("GIT_REFS = "): - f.write("GIT_REFS = 'tag: %s'\n" % __version__) - else: - f.write(line) - - class BuildPy(build_py): - def run(self): - build_py.run(self) - for pkg in self.packages: - make_version_static(self.build_lib, pkg) - - class Sdist(sdist): - def make_release_tree(self, base_dir, files): - sdist.make_release_tree(self, base_dir, files) - for pkg in self.distribution.packages: - make_version_static(base_dir, pkg) - - return dict(build_py=BuildPy, sdist=Sdist) diff --git a/src/dls_python3_skeleton/hello.py b/src/dls_python3_skeleton/hello.py deleted file mode 100644 index c0b09939..00000000 --- a/src/dls_python3_skeleton/hello.py +++ /dev/null @@ -1,41 +0,0 @@ -# The purpose of __all__ is to define the public API of this module, and which -# objects are imported if we call "from dls_python3_skeleton.hello import *" -__all__ = [ - "HelloClass", - "say_hello_lots", -] - - -class HelloClass: - """A class whose only purpose in life is to say hello""" - - def __init__(self, name: str): - """ - Args: - name: The initial value of the name of the person who gets greeted - """ - #: The name of the person who gets greeted - self.name = name - - def format_greeting(self) -> str: - """Return a greeting for `name` - - >>> HelloClass("me").format_greeting() - 'Hello me' - """ - greeting = f"Hello {self.name}" - return greeting - - -def say_hello_lots(hello: HelloClass = None, times=5): - """Print lots of greetings using the given `HelloClass` - - Args: - hello: A `HelloClass` that `format_greeting` will be called on. - If not given, use a HelloClass with name="me" - times: The number of times to call it - """ - if hello is None: - hello = HelloClass("me") - for _ in range(times): - print(hello.format_greeting()) diff --git a/src/python3_pip_skeleton/__init__.py b/src/python3_pip_skeleton/__init__.py new file mode 100644 index 00000000..0fe6655f --- /dev/null +++ b/src/python3_pip_skeleton/__init__.py @@ -0,0 +1,12 @@ +try: + # Use live version from git + from setuptools_scm import get_version + + # Warning: If the install is nested to the same depth, this will always succeed + __version__ = get_version(root="../../", relative_to=__file__) + del get_version +except (ImportError, LookupError): + # Use installed version + from ._version import __version__ + +__all__ = ["__version__"] diff --git a/src/python3_pip_skeleton/__main__.py b/src/python3_pip_skeleton/__main__.py new file mode 100644 index 00000000..1a97fb44 --- /dev/null +++ b/src/python3_pip_skeleton/__main__.py @@ -0,0 +1,16 @@ +from argparse import ArgumentParser + +from . import __version__ + +__all__ = ["main"] + + +def main(args=None): + parser = ArgumentParser() + parser.add_argument("--version", action="version", version=__version__) + args = parser.parse_args(args) + + +# test with: pipenv run python -m python3_pip_skeleton +if __name__ == "__main__": + main() diff --git a/tests/test_boilerplate_removed.py b/tests/test_boilerplate_removed.py index ca8086cb..f5204fa9 100644 --- a/tests/test_boilerplate_removed.py +++ b/tests/test_boilerplate_removed.py @@ -9,7 +9,7 @@ def skeleton_check(check: bool, text: str): - if ROOT.name == "dls-python3-skeleton": + if ROOT.name == "python3-pip-skeleton" or str(ROOT) == "/project": # In the skeleton module the check should fail check = not check text = f"Skeleton didn't raise: {text}" @@ -24,11 +24,6 @@ def assert_not_contains_text(path: str, text: str, explanation: str): skeleton_check(text in contents, f"Please change ./{path} {explanation}") -def assert_not_exists(path: str, explanation: str): - exists = (ROOT / path).exists() - skeleton_check(exists, f"Please delete ./{path} {explanation}") - - # setup.cfg def test_module_description(): conf = configparser.ConfigParser() @@ -50,30 +45,17 @@ def test_changed_README_intro(): ) -def test_changed_README_body(): +def test_removed_adopt_skeleton(): assert_not_contains_text( "README.rst", - "This is where you should put some images or code snippets", - "to include some features and why people should use it", + "This project contains template code only", + "remove the note at the start", ) -# Docs -def test_docs_ref_api_changed(): +def test_changed_README_body(): assert_not_contains_text( - "docs/reference/api.rst", - "You can mix verbose text with docstring and signature", - "to introduce the API for your module", - ) - - -def test_how_tos_written(): - assert_not_exists( - "docs/how-to/accomplish-a-task.rst", "and write some docs/how-tos" - ) - - -def test_explanations_written(): - assert_not_exists( - "docs/explanations/why-is-something-so.rst", "and write some docs/explanations" + "README.rst", + "This is where you should put some images or code snippets", + "to include some features and why people should use it", ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..2ff648c0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,9 @@ +import subprocess +import sys + +from python3_pip_skeleton import __version__ + + +def test_cli_version(): + cmd = [sys.executable, "-m", "python3_pip_skeleton", "--version"] + assert subprocess.check_output(cmd).decode().strip() == __version__ diff --git a/tests/test_dls_python3_skeleton.py b/tests/test_dls_python3_skeleton.py deleted file mode 100644 index fffef4b4..00000000 --- a/tests/test_dls_python3_skeleton.py +++ /dev/null @@ -1,28 +0,0 @@ -import subprocess -import sys - -from dls_python3_skeleton import __main__, __version__, hello - - -def test_hello_class_formats_greeting() -> None: - inst = hello.HelloClass("person") - assert inst.format_greeting() == "Hello person" - - -def test_hello_lots_defaults(capsys) -> None: - hello.say_hello_lots() - captured = capsys.readouterr() - assert captured.out == "Hello me\n" * 5 - assert captured.err == "" - - -def test_cli_greets(capsys) -> None: - __main__.main(["person", "--times=2"]) - captured = capsys.readouterr() - assert captured.out == "Hello person\n" * 2 - assert captured.err == "" - - -def test_cli_version(): - cmd = [sys.executable, "-m", "dls_python3_skeleton", "--version"] - assert subprocess.check_output(cmd).decode().strip() == __version__ From 52a0239c36920828c63207a77487f16709565a04 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 2 Sep 2022 10:39:24 +0000 Subject: [PATCH 02/50] switch org to DiamondLightSource --- CONTRIBUTING.rst | 14 +++++++------- README.rst | 22 +++++++++++----------- docs/reference.rst | 2 +- docs/tutorials/installation.rst | 4 ++-- setup.cfg | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 327e3920..8f7950d1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -7,7 +7,7 @@ filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project. -.. _GitHub: https://github.com/epics-containers/python3-pip-skeleton/issues +.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues Running the tests ----------------- @@ -17,7 +17,7 @@ To run in a container Use vscode devcontainer as follows:: - $ git clone git://github.com/epics-containers/python3-pip-skeleton.git + $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git $ vscode python3-pip-skeleton Click on 'Reopen in Container' when prompted In a vscode Terminal: @@ -30,7 +30,7 @@ To run locally Get the source source code and run the unit tests directly on your workstation as follows:: - $ git clone git://github.com/epics-containers/python3-pip-skeleton.git + $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git $ cd python3-pip-skeleton $ virtualenv .venv $ source .venv/bin/activate @@ -135,7 +135,7 @@ To make a new release, please follow this checklist: Note that tagging and pushing to the main branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/epics-containers/python3-pip-skeleton/releases +.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases Checking Dependencies @@ -153,11 +153,11 @@ Python project structure which provides a means to keep tools and techniques in sync between multiple Python projects. To update to the latest version of the skeleton, run:: - $ git pull https://github.com/dls-controls/python3-pip-skeleton main + $ git pull https://github.com/DiamondLightSource/python3-pip-skeleton main Any merge conflicts will indicate an area where something has changed that conflicts with the setup of the current module. Check the `closed pull requests -`_ +`_ of the skeleton module for more details. -.. _python3-pip-skeleton: https://epics-containers.github.io/python3-pip-skeleton +.. _python3-pip-skeleton: https://DiamondLightSource.github.io/python3-pip-skeleton diff --git a/README.rst b/README.rst index 360d0ef6..95d590c3 100644 --- a/README.rst +++ b/README.rst @@ -7,16 +7,16 @@ python3-pip-skeleton This project contains template code only. For documentation on how to adopt this skeleton project see - https://epics-containers.github.io/python3-pip-skeleton-cli + https://DiamondLightSource.github.io/python3-pip-skeleton-cli This is where you should write a short paragraph that describes what your module does, how it does it, and why people should use it. ============== ============================================================== PyPI ``pip install python3-pip-skeleton`` -Source code https://github.com/epics-containers/python3-pip-skeleton -Documentation https://epics-containers.github.io/python3-pip-skeleton -Releases https://github.com/epics-containers/python3-pip-skeleton/releases +Source code https://github.com/DiamondLightSource/python3-pip-skeleton +Documentation https://DiamondLightSource.github.io/python3-pip-skeleton +Releases https://github.com/DiamondLightSource/python3-pip-skeleton/releases ============== ============================================================== This is where you should put some images or code snippets that illustrate @@ -25,16 +25,16 @@ introductory code here. Or if it is a commandline tool then you might put some example commands here. -.. |code_ci| image:: https://github.com/epics-containers/python3-pip-skeleton/workflows/Code%20CI/badge.svg?branch=main - :target: https://github.com/epics-containers/python3-pip-skeleton/actions?query=workflow%3A%22Code+CI%22 +.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/workflows/Code%20CI/badge.svg?branch=main + :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions?query=workflow%3A%22Code+CI%22 :alt: Code CI -.. |docs_ci| image:: https://github.com/epics-containers/python3-pip-skeleton/workflows/Docs%20CI/badge.svg?branch=main - :target: https://github.com/epics-containers/python3-pip-skeleton/actions?query=workflow%3A%22Docs+CI%22 +.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/workflows/Docs%20CI/badge.svg?branch=main + :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions?query=workflow%3A%22Docs+CI%22 :alt: Docs CI -.. |coverage| image:: https://codecov.io/gh/epics-containers/python3-pip-skeleton/branch/main/graph/badge.svg - :target: https://codecov.io/gh/epics-containers/python3-pip-skeleton +.. |coverage| image:: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton/branch/main/graph/badge.svg + :target: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton :alt: Test Coverage .. |pypi_version| image:: https://img.shields.io/pypi/v/python3-pip-skeleton.svg @@ -49,4 +49,4 @@ Or if it is a commandline tool then you might put some example commands here. Anything below this line is used when viewing README.rst and will be replaced when included in index.rst -See https://epics-containers.github.io/python3-pip-skeleton for more detailed documentation. +See https://DiamondLightSource.github.io/python3-pip-skeleton for more detailed documentation. diff --git a/docs/reference.rst b/docs/reference.rst index bfa7a4f4..84c8cf13 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -9,7 +9,7 @@ Technical reference material including APIs and release notes. :caption: Reference reference/api - Releases + Releases Index .. diff --git a/docs/tutorials/installation.rst b/docs/tutorials/installation.rst index bbffd015..399dc2c5 100644 --- a/docs/tutorials/installation.rst +++ b/docs/tutorials/installation.rst @@ -30,7 +30,7 @@ You can now use ``pip`` to install the library:: If you require a feature that is not currently released you can also install from github:: - python3 -m pip install git+https://github.com/epics-containers/python3-pip-skeleton.git + python3 -m pip install git+https://github.com/DiamondLightSource/python3-pip-skeleton.git The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing:: @@ -42,4 +42,4 @@ Running in a container To pull the container from github container registry and run:: - docker run ghcr.io/epics-containers/python3-pip-skeleton:main --version + docker run ghcr.io/DiamondLightSource/python3-pip-skeleton:main --version diff --git a/setup.cfg b/setup.cfg index 008fd8a4..f32a3441 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = python3-pip-skeleton description = One line description of your module -url = https://github.com/epics-containers/python3-pip-skeleton +url = https://github.com/DiamondLightSource/python3-pip-skeleton author = Firstname Lastname author_email = email@address.com license = Apache License 2.0 From 13da3015fba3f486b0f784449a87982c491e9bf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:39:50 +0000 Subject: [PATCH 03/50] Bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code.yml | 6 +++--- .github/workflows/docs.yml | 2 +- .github/workflows/docs_clean.yml | 2 +- .github/workflows/linkcheck.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 5ea6e1f2..b2f561a7 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python uses: actions/setup-python@v4 @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -73,7 +73,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c3c5eda8..09e6f3ef 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: run: sudo apt-get install graphviz - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index 2059c7f3..f39519f7 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -18,7 +18,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: gh-pages diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 6b37f194..e6838560 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 From 0f37bbc885ae54486d890deee65547ae3cc10290 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:28:58 +0000 Subject: [PATCH 04/50] Bump black from 22.6.0 to 22.8.0 Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f32a3441..568ecc48 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ setup_requires = [options.extras_require] # For development tests/docs dev = - black==22.6.0 + black==22.8.0 flake8-isort isort>5.0 mypy From 9e16fc2ce9bd9a318b19c31107aeb4d3c64b3811 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 2 Sep 2022 12:50:57 +0000 Subject: [PATCH 05/50] fix twine command --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index b2f561a7..ebffb2fc 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -174,4 +174,4 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: pipx run twine upload dist/*/whl dist/*.tar.gz + run: pipx run twine upload dist/*.whl dist/*.tar.gz From c975f6dfc55dffc395099ba7ae2b9ad28e136814 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 2 Sep 2022 15:26:48 +0000 Subject: [PATCH 06/50] obscure the docs cleanup email --- .github/workflows/docs_clean.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index f39519f7..e0f7e485 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -36,6 +36,6 @@ jobs: rm -r ${{ env.remove_me }} sed -i /${{ env.remove_me }}/d versions.txt git config --global user.name 'GitHub Actions Docs Cleanup CI' - git config --global user.email 'gha@users.noreply.github.com' + git config --global user.email 'GithubActionsCleanup@users.noreply.github.com' git commit -am"removing redundant docs version ${{ env.remove_me }}" git push From 10e23967dbdd36a2e207a1b28f1d79f201dff262 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 9 Sep 2022 14:21:46 +0000 Subject: [PATCH 07/50] Made tox faster with tox-direct --- Dockerfile | 1 + setup.cfg | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76b5dad3..056d144e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN cd /project && \ RUN python -m venv /venv ENV PATH=/venv/bin:$PATH +ENV TOX_DIRECT=1 RUN cd /project && \ pip install --upgrade pip && \ diff --git a/setup.cfg b/setup.cfg index 568ecc48..5d9ac269 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ dev = setuptools_scm[toml]>=6.2 sphinx-rtd-theme-github-versions tox + tox-direct types-mock [options.packages.find] @@ -75,9 +76,8 @@ extend-ignore = F811, # support typing.overload decorator F722, # allow Annotated[typ, some_func("some string")] exclude = - ui_* .tox - .venv + venv [tool:pytest] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error @@ -103,22 +103,22 @@ source = # NOTE that we pre-install all tools in the dev dependencies (including tox). # Hence the use of allowlist_externals instead of using the tox virtualenvs. # This ensures a match between developer time tools in the IDE and tox tools. +# Setting TOX_DIRECT=1 in the environment will make this even faster [tox:tox] -minversion = 3.7 -skipsdist=true -skipinstall=true - -[testenv:{pre-commit,mypy,pytest,docs}] -passenv = - PYTHONPATH - HOME -allowlist_externals = - pytest - pre-commit - mypy - sphinx-build -commands = - pytest: pytest tests {posargs} - mypy: mypy src tests {posargs} - pre-commit: pre-commit run --all-files {posargs} - docs: sphinx-build -EWT --keep-going docs build/html {posargs} +skipsdist = True + +[testenv:pytest] +allowlist_externals = pytest +commands = pytest {posargs} + +[testenv:mypy] +allowlist_externals = mypy +commands = mypy src tests {posargs} + +[testenv:pre-commit] +allowlist_externals = pre-commit +commands = pre-commit run --all-files {posargs} + +[testenv:docs] +allowlist_externals = sphinx-build +commands = sphinx-build -EWT --keep-going docs build/html {posargs} From d7a937a22bdf6c95c2ae9a49dfbce9e9509e1c9f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 9 Sep 2022 15:45:31 +0000 Subject: [PATCH 08/50] Switch to pydata-theme and split docs There are now developer and user guides. The version switcher is native to the theme so added a script to generate it. --- .github/CONTRIBUTING.rst | 35 ++++ .github/pages/make_switcher.py | 99 +++++++++++ .github/workflows/code.yml | 5 +- .github/workflows/docs.yml | 6 +- .github/workflows/docs_clean.yml | 7 +- .vscode/extensions.json | 4 +- CONTRIBUTING.rst | 163 ------------------ Dockerfile | 4 +- README.rst | 16 +- docs/_static/theme_overrides.css | 34 ---- docs/conf.py | 80 +++++++-- .../explanations/decisions.rst | 0 .../0001-record-architecture-decisions.rst | 0 docs/developer/how-to/build-docs.rst | 20 +++ docs/developer/how-to/contribute.rst | 1 + docs/developer/how-to/lint.rst | 38 ++++ docs/developer/how-to/make-release.rst | 16 ++ docs/developer/how-to/run-tests.rst | 12 ++ docs/developer/how-to/static-analysis.rst | 8 + docs/developer/how-to/update-tools.rst | 16 ++ docs/developer/index.rst | 62 +++++++ docs/developer/reference/standards.rst | 64 +++++++ docs/developer/tutorials/dev-install.rst | 60 +++++++ docs/explanations.rst | 11 -- docs/genindex.rst | 5 + docs/how-to.rst | 11 -- docs/how-to/contributing.rst | 1 - docs/index.rst | 51 ++---- docs/reference.rst | 17 -- docs/tutorials.rst | 11 -- docs/user/explanations/docs-structure.rst | 18 ++ docs/user/how-to/run-container.rst | 15 ++ docs/user/index.rst | 57 ++++++ docs/{ => user}/reference/api.rst | 0 docs/{ => user}/tutorials/installation.rst | 21 +-- setup.cfg | 6 +- src/python3_pip_skeleton/__main__.py | 2 +- 37 files changed, 640 insertions(+), 336 deletions(-) create mode 100644 .github/CONTRIBUTING.rst create mode 100755 .github/pages/make_switcher.py delete mode 100644 CONTRIBUTING.rst delete mode 100644 docs/_static/theme_overrides.css rename docs/{ => developer}/explanations/decisions.rst (100%) rename docs/{ => developer}/explanations/decisions/0001-record-architecture-decisions.rst (100%) create mode 100644 docs/developer/how-to/build-docs.rst create mode 100644 docs/developer/how-to/contribute.rst create mode 100644 docs/developer/how-to/lint.rst create mode 100644 docs/developer/how-to/make-release.rst create mode 100644 docs/developer/how-to/run-tests.rst create mode 100644 docs/developer/how-to/static-analysis.rst create mode 100644 docs/developer/how-to/update-tools.rst create mode 100644 docs/developer/index.rst create mode 100644 docs/developer/reference/standards.rst create mode 100644 docs/developer/tutorials/dev-install.rst delete mode 100644 docs/explanations.rst create mode 100644 docs/genindex.rst delete mode 100644 docs/how-to.rst delete mode 100644 docs/how-to/contributing.rst delete mode 100644 docs/reference.rst delete mode 100644 docs/tutorials.rst create mode 100644 docs/user/explanations/docs-structure.rst create mode 100644 docs/user/how-to/run-container.rst create mode 100644 docs/user/index.rst rename docs/{ => user}/reference/api.rst (100%) rename docs/{ => user}/tutorials/installation.rst (56%) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst new file mode 100644 index 00000000..19ab494f --- /dev/null +++ b/.github/CONTRIBUTING.rst @@ -0,0 +1,35 @@ +Contributing to the project +=========================== + +Contributions and issues are most welcome! All issues and pull requests are +handled through GitHub_. Also, please check for any existing issues before +filing a new one. If you have a great idea but it involves big changes, please +file a ticket before making a pull request! We want to make sure you don't spend +your time coding something that might not fit the scope of the project. + +.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues + +Issue or Discussion? +-------------------- + +Github also offers discussions_ as a place to ask questions and share ideas. If +your issue is open ended and it is not obvious when it can be "closed", please +raise it as a discussion instead. + +.. _discussions: https://github.com/DiamondLightSource/python3-pip-skeleton/discussions + +Code coverage +------------- + +While 100% code coverage does not make a library bug-free, it significantly +reduces the number of easily caught bugs! Please make sure coverage remains the +same or is improved by a pull request! + +Developer guide +--------------- + +The `Developer Guide`_ contains information on setting up a development +environment, running the tests and what standards the code and documentation +should follow. + +.. _Developer Guide: https://diamondlightsource.github.io/python3-pip-skeleton/main/developer/how-to/contribute.html diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py new file mode 100755 index 00000000..5c65d788 --- /dev/null +++ b/.github/pages/make_switcher.py @@ -0,0 +1,99 @@ +import json +import logging +from argparse import ArgumentParser +from pathlib import Path +from subprocess import CalledProcessError, check_output +from typing import List, Optional + + +def report_output(stdout: bytes, label: str) -> List[str]: + ret = stdout.decode().strip().split("\n") + print(f"{label}: {ret}") + return ret + + +def get_branch_contents(ref: str) -> List[str]: + """Get the list of directories in a branch.""" + stdout = check_output(["git", "ls-tree", "-d", "--name-only", ref]) + return report_output(stdout, "Branch contents") + + +def get_sorted_tags_list() -> List[str]: + """Get a list of sorted tags in descending order from the repository.""" + stdout = check_output(["git", "tag", "-l", "--sort=-v:refname"]) + return report_output(stdout, "Tags list") + + +def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[str]: + """Generate the file containing the list of all GitHub Pages builds.""" + # Get the directories (i.e. builds) from the GitHub Pages branch + try: + builds = set(get_branch_contents(ref)) + except CalledProcessError: + builds = set() + logging.warning(f"Cannot get {ref} contents") + + # Add and remove from the list of builds + if add: + builds.add(add) + if remove: + assert remove in builds, f"Build '{remove}' not in {sorted(builds)}" + builds.remove(remove) + + # Get a sorted list of tags + tags = get_sorted_tags_list() + + # Make the sorted versions list from main branches and tags + versions: List[str] = [] + for version in ["master", "main"] + tags: + if version in builds: + versions.append(version) + builds.remove(version) + + # Add in anything that is left to the bottom + versions += sorted(builds) + print(f"Sorted versions: {versions}") + return versions + + +def write_json(path: Path, repository: str, versions: str): + org, repo_name = repository.split("/") + struct = [ + dict(name=version, url=f"https://{org}.github.io/{repo_name}/{version}/") + for version in versions + ] + text = json.dumps(struct, indent=2) + print(f"JSON switcher:\n{text}") + path.write_text(text) + + +def main(args=None): + parser = ArgumentParser( + description="Make a versions.txt file from gh-pages directories" + ) + parser.add_argument( + "--add", + help="Add this directory to the list of existing directories", + ) + parser.add_argument( + "--remove", + help="Remove this directory from the list of existing directories", + ) + parser.add_argument( + "repository", + help="The GitHub org and repository name: ORG/REPO", + ) + parser.add_argument( + "output", + type=Path, + help="Path of write switcher.json to", + ) + args = parser.parse_args(args) + + # Write the versions file + versions = get_versions("origin/gh-pages", args.add, args.remove) + write_json(args.output, args.repository, versions) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index ebffb2fc..0aae7fb5 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -133,7 +133,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: dist - path: dist/* + path: dist sdist: needs: container @@ -164,8 +164,7 @@ jobs: uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14 with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} - files: | - dist/* + files: dist/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 09e6f3ef..a684d031 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,10 +43,10 @@ jobs: - name: Move to versioned directory # e.g. main or 0.1.2 - run: mv build/html ".github/pages/${GITHUB_REF##*/}" + run: mv build/html ".github/pages/${{ github.ref_name }}" - - name: Write versions.txt - run: sphinx_rtd_theme_github_versions .github/pages + - name: Write switcher.json + run: python .github/pages/make_switcher.py --add "${{ github.ref_name }}" ${{ github.repository }} .github/pages/switcher.json - name: Publish Docs to gh-pages if: github.event_name == 'push' diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index e0f7e485..b80e4c22 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -32,10 +32,9 @@ jobs: - name: update index and push changes run: | - echo removing redundant documentation version ${{ env.remove_me }} rm -r ${{ env.remove_me }} - sed -i /${{ env.remove_me }}/d versions.txt + python make_switcher.py --remove ${{ env.remove_me }} ${{ github.repository }} switcher.json git config --global user.name 'GitHub Actions Docs Cleanup CI' - git config --global user.email 'GithubActionsCleanup@users.noreply.github.com' - git commit -am"removing redundant docs version ${{ env.remove_me }}" + git config --global user.email 'GithubActionsCleanup@noreply.github.com' + git commit -am"removing redundant docs version ${{ env.remove_me }}" git push diff --git a/.vscode/extensions.json b/.vscode/extensions.json index d173f8d6..041f8944 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,8 +1,8 @@ { "recommendations": [ - "ms-vscode-remote.remote-containers" + "ms-vscode-remote.remote-containers", "ms-python.vscode-pylance", "ms-python.python", "ryanluker.vscode-coverage-gutters" ] -} +} \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 8f7950d1..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,163 +0,0 @@ -Contributing -============ - -Contributions and issues are most welcome! All issues and pull requests are -handled through GitHub_. Also, please check for any existing issues before -filing a new one. If you have a great idea but it involves big changes, please -file a ticket before making a pull request! We want to make sure you don't spend -your time coding something that might not fit the scope of the project. - -.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues - -Running the tests ------------------ - -To run in a container -~~~~~~~~~~~~~~~~~~~~~ - -Use vscode devcontainer as follows:: - - $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git - $ vscode python3-pip-skeleton - Click on 'Reopen in Container' when prompted - In a vscode Terminal: - $ tox -p - - -To run locally -~~~~~~~~~~~~~~ - -Get the source source code and run the unit tests directly -on your workstation as follows:: - - $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git - $ cd python3-pip-skeleton - $ virtualenv .venv - $ source .venv/bin/activate - $ pip install -e .[dev] - $ tox -p - -In both cases tox -p runs in parallel the following checks: - - - Build Sphinx Documentation - - run pytest on all tests in ./tests - - run mypy linting on all files in ./src ./tests - - run pre-commit checks: - - - run flake8 style checks against all source - - run black formatting checks against all source - -While 100% code coverage does not make a library bug-free, it significantly -reduces the number of easily caught bugs! Please make sure coverage remains the -same or is improved by a pull request! - -Code Styling ------------- - -The code in this repository conforms to standards set by the following tools: - -- black_ for code formatting -- flake8_ for style checks -- isort_ for import ordering -- mypy_ for static type checking - -flake8 and black and isort are run by pre-commit_. You can run the above checks on -all files with this command:: - - $ tox -e pre-commit,mypy - -Or you can install a pre-commit hook that will run each time you do a ``git -commit`` on just the files that have changed. Note that mypy is not in -the pre-commit because it is a little slow :: - - $ pre-commit install - -.. _black: https://github.com/psf/black -.. _flake8: https://flake8.pycqa.org/en/latest/ -.. _isort: https://github.com/PyCQA/isort -.. _mypy: https://github.com/python/mypy -.. _pre-commit: https://pre-commit.com/ - -Docstrings are pre-processed using the Sphinx Napoleon extension. As such, -google-style_ is considered as standard for this repository. Please use type -hints in the function signature for types. For example:: - - def func(arg1: str, arg2: int) -> bool: - """Summary line. - - Extended description of function. - - Args: - arg1: Description of arg1 - arg2: Description of arg2 - - Returns: - Description of return value - """ - return True - -.. _google-style: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/index.html#google-vs-numpy - -Documentation -------------- - -Documentation is contained in the ``docs`` directory and extracted from -docstrings of the API. - -Docs follow the underlining convention:: - - Headling 1 (page title) - ======================= - - Heading 2 - --------- - - Heading 3 - ~~~~~~~~~ - -You can build the docs from the project directory by running:: - - $ tox -e docs - $ firefox build/html/index.html - -Release Process ---------------- - -To make a new release, please follow this checklist: - -- Choose a new PEP440 compliant release number (see https://peps.python.org/pep-0440/) -- Go to the GitHub release_ page -- Choose ``Draft New Release`` -- Click ``Choose Tag`` and supply the new tag you chose (click create new tag) -- Click ``Generate release notes``, review and edit these notes -- Choose a title and click ``Publish Release`` - -Note that tagging and pushing to the main branch has the same effect except that -you will not get the option to edit the release notes. - -.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases - - -Checking Dependencies ---------------------- - -To see a graph of the python package dependency tree type:: - - pipdeptree - -Updating the tools ------------------- - -This module is merged with the python3-pip-skeleton_. This is a generic -Python project structure which provides a means to keep tools and -techniques in sync between multiple Python projects. To update to the -latest version of the skeleton, run:: - - $ git pull https://github.com/DiamondLightSource/python3-pip-skeleton main - -Any merge conflicts will indicate an area where something has changed that -conflicts with the setup of the current module. Check the `closed pull requests -`_ -of the skeleton module for more details. - -.. _python3-pip-skeleton: https://DiamondLightSource.github.io/python3-pip-skeleton diff --git a/Dockerfile b/Dockerfile index 056d144e..b8bfe727 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # This file is for use as a devcontainer and a runtime container -# -# The devcontainer should use the build target and run as root with podman +# +# The devcontainer should use the build target and run as root with podman # or docker with user namespaces. # FROM python:3.10 as build diff --git a/README.rst b/README.rst index 95d590c3..0450bd2b 100644 --- a/README.rst +++ b/README.rst @@ -4,9 +4,9 @@ python3-pip-skeleton |code_ci| |docs_ci| |coverage| |pypi_version| |license| .. note:: - + This project contains template code only. For documentation on how to - adopt this skeleton project see + adopt this skeleton project see https://DiamondLightSource.github.io/python3-pip-skeleton-cli This is where you should write a short paragraph that describes what your module does, @@ -21,9 +21,17 @@ Releases https://github.com/DiamondLightSource/python3-pip-skeleton/releas This is where you should put some images or code snippets that illustrate some relevant examples. If it is a library then you might put some -introductory code here. +introductory code here: + +.. code-block:: python + + from python3_pip_skeleton import __version__ + + print(f"Hello python3_pip_skeleton {__version__}") + +Or if it is a commandline tool then you might put some example commands here:: -Or if it is a commandline tool then you might put some example commands here. + $ python -m python3_pip_skeleton --version .. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/workflows/Code%20CI/badge.svg?branch=main :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions?query=workflow%3A%22Code+CI%22 diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css deleted file mode 100644 index 5fd9b721..00000000 --- a/docs/_static/theme_overrides.css +++ /dev/null @@ -1,34 +0,0 @@ -/* override table width restrictions */ -@media screen and (min-width: 639px) { - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from - overriding this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - } -} - -/* override table padding */ -.rst-content table.docutils th, .rst-content table.docutils td { - padding: 4px 6px; -} - -/* Add two-column option */ -@media only screen and (min-width: 1000px) { - .columns { - padding-left: 10px; - padding-right: 10px; - float: left; - width: 50%; - min-height: 145px; - } -} - -.endcolumns { - clear: both -} - -/* Hide toctrees within columns and captions from all toctrees. - This is what makes the include trick in index.rst work */ -.columns .toctree-wrapper, .toctree-wrapper .caption-text { - display: none; -} diff --git a/docs/conf.py b/docs/conf.py index bee41d4b..0b6dd5bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,9 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from pathlib import Path +from subprocess import check_output + import python3_pip_skeleton # -- General configuration ------------------------------------------------ @@ -16,9 +19,12 @@ # The short X.Y version. if "+" in release: - # Not on a tag - version = "main" + # Not on a tag, use branch name + root = Path(__file__).absolute().parent.parent + git_branch = check_output("git branch --show-current".split(), cwd=root) + version = git_branch.decode().strip() else: + branch = "main" version = release extensions = [ @@ -32,6 +38,10 @@ "sphinx.ext.viewcode", # Adds the inheritance-diagram generation directive "sphinx.ext.inheritance_diagram", + # Add a copy button to each code block + "sphinx_copybutton", + # For the card element + "sphinx_design", ] # If true, Sphinx will warn about all references where the target cannot @@ -50,7 +60,6 @@ ("py:class", "'bool'"), ("py:class", "'object'"), ("py:class", "'id'"), - ("py:class", "apischema.utils.UndefinedType"), ("py:class", "typing_extensions.Literal"), ] @@ -94,27 +103,65 @@ # Common links that should be available on every page rst_epilog = """ -.. _Diamond Light Source: - http://www.diamond.ac.uk +.. _Diamond Light Source: http://www.diamond.ac.uk +.. _black: https://github.com/psf/black +.. _flake8: https://flake8.pycqa.org/en/latest/ +.. _isort: https://github.com/PyCQA/isort +.. _mypy: http://mypy-lang.org/ +.. _pre-commit: https://pre-commit.com/ """ -# Ignore localhost links for period check that links in docs are valid +# Ignore localhost links for periodic check that links in docs are valid linkcheck_ignore = [r"http://localhost:\d+/"] +# Set copy-button to ignore python and bash prompts +# https://sphinx-copybutton.readthedocs.io/en/latest/use.html#using-regexp-prompt-identifiers +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme_github_versions" - -# Options for the sphinx rtd theme, use DLS blue -html_theme_options = dict(style_nav_header_background="rgb(7, 43, 93)") - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_theme = "pydata_sphinx_theme" +github_repo = project +github_user = "DiamondLightSource" + +# Theme options for pydata_sphinx_theme +html_theme_options = dict( + logo=dict( + text=project, + ), + use_edit_page_button=True, + github_url=f"https://github.com/{github_user}/{github_repo}", + icon_links=[ + dict( + name="PyPI", + url=f"https://pypi.org/project/{project}", + icon="fas fa-cube", + ) + ], + switcher=dict( + json_url=f"https://{github_user}.github.io/{github_repo}/switcher.json", + version_match=version, + ), + navbar_end=["theme-switcher", "icon-links", "version-switcher"], + external_links=[ + dict( + name="Release Notes", + url=f"https://github.com/{github_user}/{github_repo}/releases", + ) + ], +) + +# A dictionary of values to pass into the template engine’s context for all pages +html_context = dict( + github_user=github_user, + github_repo=project, + github_version=version, + doc_path="docs", +) # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False @@ -122,9 +169,6 @@ # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False -# Add some CSS classes for columns and other tweaks in a custom css file -html_css_files = ["theme_overrides.css"] - # Logo html_logo = "images/dls-logo.svg" html_favicon = "images/dls-favicon.ico" diff --git a/docs/explanations/decisions.rst b/docs/developer/explanations/decisions.rst similarity index 100% rename from docs/explanations/decisions.rst rename to docs/developer/explanations/decisions.rst diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.rst b/docs/developer/explanations/decisions/0001-record-architecture-decisions.rst similarity index 100% rename from docs/explanations/decisions/0001-record-architecture-decisions.rst rename to docs/developer/explanations/decisions/0001-record-architecture-decisions.rst diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst new file mode 100644 index 00000000..9540de1c --- /dev/null +++ b/docs/developer/how-to/build-docs.rst @@ -0,0 +1,20 @@ +Build the docs using sphinx +=========================== + +You can build the `sphinx`_ based docs from the project directory by running:: + + $ tox -e docs + +This will build the static docs on the ``docs`` directory, which includes API +docs that pull in docstrings from the code. + +.. seealso:: + + `documentation_standards` + +The docs will be built into the ``build/html`` directory, and can be opened +locally with a web browse:: + + $ firefox build/html/index.html + +.. _sphinx: https://www.sphinx-doc.org/ \ No newline at end of file diff --git a/docs/developer/how-to/contribute.rst b/docs/developer/how-to/contribute.rst new file mode 100644 index 00000000..65b992f0 --- /dev/null +++ b/docs/developer/how-to/contribute.rst @@ -0,0 +1 @@ +.. include:: ../../../.github/CONTRIBUTING.rst diff --git a/docs/developer/how-to/lint.rst b/docs/developer/how-to/lint.rst new file mode 100644 index 00000000..1086c3c4 --- /dev/null +++ b/docs/developer/how-to/lint.rst @@ -0,0 +1,38 @@ +Run linting using pre-commit +============================ + +Code linting is handled by black_, flake8_ and isort_ run under pre-commit_. + +Running pre-commit +------------------ + +You can run the above checks on all files with this command:: + + $ tox -e pre-commit + +Or you can install a pre-commit hook that will run each time you do a ``git +commit`` on just the files that have changed:: + + $ pre-commit install + +Fixing issues +------------- + +If black reports an issue you can tell it to reformat all the files in the +repository:: + + $ black . + +Likewise with isort:: + + $ isort . + +If you get any flake8 issues you will have to fix those manually. + +VSCode support +-------------- + +The ``.vscode/settings.json`` will run black and isort formatters as well as +flake8 checking on save. Issues will be highlighted in the editor window. + + diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst new file mode 100644 index 00000000..747e44a2 --- /dev/null +++ b/docs/developer/how-to/make-release.rst @@ -0,0 +1,16 @@ +Make a release +============== + +To make a new release, please follow this checklist: + +- Choose a new PEP440 compliant release number (see https://peps.python.org/pep-0440/) +- Go to the GitHub release_ page +- Choose ``Draft New Release`` +- Click ``Choose Tag`` and supply the new tag you chose (click create new tag) +- Click ``Generate release notes``, review and edit these notes +- Choose a title and click ``Publish Release`` + +Note that tagging and pushing to the main branch has the same effect except that +you will not get the option to edit the release notes. + +.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases \ No newline at end of file diff --git a/docs/developer/how-to/run-tests.rst b/docs/developer/how-to/run-tests.rst new file mode 100644 index 00000000..d2e03644 --- /dev/null +++ b/docs/developer/how-to/run-tests.rst @@ -0,0 +1,12 @@ +Run the tests using pytest +========================== + +Testing is done with pytest_. It will find functions in the project that `look +like tests`_, and run them to check for errors. You can run it with:: + + $ tox -e pytest + +It will also report coverage to the commandline and to ``cov.xml``. + +.. _pytest: https://pytest.org/ +.. _look like tests: https://docs.pytest.org/explanation/goodpractices.html#test-discovery diff --git a/docs/developer/how-to/static-analysis.rst b/docs/developer/how-to/static-analysis.rst new file mode 100644 index 00000000..065920e1 --- /dev/null +++ b/docs/developer/how-to/static-analysis.rst @@ -0,0 +1,8 @@ +Run static analysis using mypy +============================== + +Static type analysis is done with mypy_. It checks type definition in source +files without running them, and highlights potential issues where types do not +match. You can run it with:: + + $ tox -e mypy diff --git a/docs/developer/how-to/update-tools.rst b/docs/developer/how-to/update-tools.rst new file mode 100644 index 00000000..c1075ee8 --- /dev/null +++ b/docs/developer/how-to/update-tools.rst @@ -0,0 +1,16 @@ +Update the tools +================ + +This module is merged with the python3-pip-skeleton_. This is a generic +Python project structure which provides a means to keep tools and +techniques in sync between multiple Python projects. To update to the +latest version of the skeleton, run:: + + $ git pull --rebase=false https://github.com/DiamondLightSource/python3-pip-skeleton + +Any merge conflicts will indicate an area where something has changed that +conflicts with the setup of the current module. Check the `closed pull requests +`_ +of the skeleton module for more details. + +.. _python3-pip-skeleton: https://DiamondLightSource.github.io/python3-pip-skeleton diff --git a/docs/developer/index.rst b/docs/developer/index.rst new file mode 100644 index 00000000..bf291875 --- /dev/null +++ b/docs/developer/index.rst @@ -0,0 +1,62 @@ +Developer Guide +=============== + +Documentation is split into four categories, also accessible from links in the +side-bar. + +.. grid:: 2 + :gutter: 4 + + .. grid-item-card:: :material-regular:`directions_run;3em` + + .. toctree:: + :caption: Tutorials + :maxdepth: 1 + + tutorials/dev-install + + +++ + + Tutorials for getting up and running as a developer. + + .. grid-item-card:: :material-regular:`task;3em` + + .. toctree:: + :caption: How-to Guides + :maxdepth: 1 + + how-to/contribute + how-to/build-docs + how-to/run-tests + how-to/static-analysis + how-to/lint + how-to/update-tools + how-to/make-release + + +++ + + Practical step-by-step guides for day-to-day dev tasks. + + .. grid-item-card:: :material-regular:`apartment;3em` + + .. toctree:: + :caption: Explanations + :maxdepth: 1 + + explanations/decisions + + +++ + + Explanations of how and why the architecture is why it is. + + .. grid-item-card:: :material-regular:`description;3em` + + .. toctree:: + :caption: Reference + :maxdepth: 1 + + reference/standards + + +++ + + Technical reference material on standards in use. diff --git a/docs/developer/reference/standards.rst b/docs/developer/reference/standards.rst new file mode 100644 index 00000000..b78a719e --- /dev/null +++ b/docs/developer/reference/standards.rst @@ -0,0 +1,64 @@ +Standards +========= + +This document defines the code and documentation standards used in this +repository. + +Code Standards +-------------- + +The code in this repository conforms to standards set by the following tools: + +- black_ for code formatting +- flake8_ for style checks +- isort_ for import ordering +- mypy_ for static type checking + +.. seealso:: + + How-to guides `../how-to/lint` and `../how-to/static-analysis` + +.. _documentation_standards: + +Documentation Standards +----------------------- + +Docstrings are pre-processed using the Sphinx Napoleon extension. As such, +google-style_ is considered as standard for this repository. Please use type +hints in the function signature for types. For example: + +.. code:: python + + def func(arg1: str, arg2: int) -> bool: + """Summary line. + + Extended description of function. + + Args: + arg1: Description of arg1 + arg2: Description of arg2 + + Returns: + Description of return value + """ + return True + +.. _google-style: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/index.html#google-vs-numpy + +Documentation is contained in the ``docs`` directory and extracted from +docstrings of the API. + +Docs follow the underlining convention:: + + Headling 1 (page title) + ======================= + + Heading 2 + --------- + + Heading 3 + ~~~~~~~~~ + +.. seealso:: + + How-to guide `../how-to/build-docs` \ No newline at end of file diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst new file mode 100644 index 00000000..c2632683 --- /dev/null +++ b/docs/developer/tutorials/dev-install.rst @@ -0,0 +1,60 @@ +Developer install +================= + +These instructions will take you through the minimal steps required to get a dev +environment setup, so you can run the tests locally. + +Clone the repository +-------------------- + +First clone the repository locally using `Git +`_:: + + $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git + +Install dependencies +-------------------- + +You can choose to either develop on the host machine using a `venv` (which +requires python 3.8 or later) or to run in a container under `VSCode +`_ + +.. tab-set:: + + .. tab-item:: Local virtualenv + + .. code:: + + $ cd python3-pip-skeleton + $ python3 -m venv venv + $ source venv/bin/activate + $ pip install -e .[dev] + + .. tab-item:: VSCode devcontainer + + .. code:: + + $ vscode python3-pip-skeleton + # Click on 'Reopen in Container' when prompted + # Open a new terminal + +See what was installed +---------------------- + +To see a graph of the python package dependency tree type:: + + $ pipdeptree + +Build and test +-------------- + +Now you have a development environment you can run the tests in a terminal:: + + $ tox -p + +This will run in parallel the following checks: + +- `../how-to/build-docs` +- `../how-to/run-tests` +- `../how-to/static-analysis` +- `../how-to/lint` diff --git a/docs/explanations.rst b/docs/explanations.rst deleted file mode 100644 index 1e329673..00000000 --- a/docs/explanations.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -Explanations -============ - -Explanation of how the library works and why it works that way. - -.. toctree:: - :caption: Explanations - - explanations/decisions diff --git a/docs/genindex.rst b/docs/genindex.rst new file mode 100644 index 00000000..93eb8b29 --- /dev/null +++ b/docs/genindex.rst @@ -0,0 +1,5 @@ +API Index +========= + +.. + https://stackoverflow.com/a/42310803 diff --git a/docs/how-to.rst b/docs/how-to.rst deleted file mode 100644 index 700797cc..00000000 --- a/docs/how-to.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -How-to Guides -============= - -Practical step-by-step guides for the more experienced user. - -.. toctree:: - :caption: How-to Guides - - how-to/contributing diff --git a/docs/how-to/contributing.rst b/docs/how-to/contributing.rst deleted file mode 100644 index ac7b6bcf..00000000 --- a/docs/how-to/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 9bde8adf..df33c8e6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,48 +1,29 @@ +:html_theme.sidebar_secondary.remove: + .. include:: ../README.rst :end-before: when included in index.rst - How the documentation is structured ----------------------------------- -Documentation is split into four categories, also accessible from links in the -side-bar. - -.. rst-class:: columns - -`tutorials` -~~~~~~~~~~~ - -.. include:: tutorials.rst - :start-after: ========= - -.. rst-class:: columns - -`how-to` -~~~~~~~~ - -.. include:: how-to.rst - :start-after: ============= - -.. rst-class:: columns - -`explanations` -~~~~~~~~~~~~~~ +The documentation is split into 2 sections: -.. include:: explanations.rst - :start-after: ============ +.. grid:: 2 -.. rst-class:: columns + .. grid-item-card:: :material-regular:`person;4em` + :link: user/index + :link-type: doc -`reference` -~~~~~~~~~~~ + The User Guide contains documentation on how to install and use python3-pip-skeleton. -.. include:: reference.rst - :start-after: ========= + .. grid-item-card:: :material-regular:`code;4em` + :link: developer/index + :link-type: doc -.. rst-class:: endcolumns + The Developer Guide contains documentation on how to develop and contribute changes back to python3-pip-skeleton. -About the documentation -~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :hidden: -`Why is the documentation structured this way? `_ + user/index + developer/index diff --git a/docs/reference.rst b/docs/reference.rst deleted file mode 100644 index 84c8cf13..00000000 --- a/docs/reference.rst +++ /dev/null @@ -1,17 +0,0 @@ -:orphan: - -Reference -========= - -Technical reference material including APIs and release notes. - -.. toctree:: - :caption: Reference - - reference/api - Releases - Index - -.. - Index link above is a hack to make genindex.html a relative link - https://stackoverflow.com/a/31820846 diff --git a/docs/tutorials.rst b/docs/tutorials.rst deleted file mode 100644 index dfdef509..00000000 --- a/docs/tutorials.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -Tutorials -========= - -Tutorials for installation, library and commandline usage. New users start here. - -.. toctree:: - :caption: Tutorials - - tutorials/installation diff --git a/docs/user/explanations/docs-structure.rst b/docs/user/explanations/docs-structure.rst new file mode 100644 index 00000000..f25a09ba --- /dev/null +++ b/docs/user/explanations/docs-structure.rst @@ -0,0 +1,18 @@ +About the documentation +----------------------- + + :material-regular:`format_quote;2em` + + The Grand Unified Theory of Documentation + + -- David Laing + +There is a secret that needs to be understood in order to write good software +documentation: there isn't one thing called *documentation*, there are four. + +They are: *tutorials*, *how-to guides*, *technical reference* and *explanation*. +They represent four different purposes or functions, and require four different +approaches to their creation. Understanding the implications of this will help +improve most documentation - often immensely. + +`More information on this topic. `_ diff --git a/docs/user/how-to/run-container.rst b/docs/user/how-to/run-container.rst new file mode 100644 index 00000000..84f857af --- /dev/null +++ b/docs/user/how-to/run-container.rst @@ -0,0 +1,15 @@ +Run in a container +================== + +Pre-built containers with python3-pip-skeleton and its dependencies already +installed are available on `Github Container Registry +`_. + +Starting the container +---------------------- + +To pull the container from github container registry and run:: + + $ docker run ghcr.io/DiamondLightSource/python3-pip-skeleton:main --version + +To get a released version, use a numbered release instead of ``main``. diff --git a/docs/user/index.rst b/docs/user/index.rst new file mode 100644 index 00000000..2c94a0c0 --- /dev/null +++ b/docs/user/index.rst @@ -0,0 +1,57 @@ +User Guide +========== + +Documentation is split into four categories, also accessible from links in the +side-bar. + +.. grid:: 2 + :gutter: 4 + + .. grid-item-card:: :material-regular:`directions_walk;3em` + + .. toctree:: + :caption: Tutorials + :maxdepth: 1 + + tutorials/installation + + +++ + + Tutorials for installation and typical usage. New users start here. + + .. grid-item-card:: :material-regular:`directions;3em` + + .. toctree:: + :caption: How-to Guides + :maxdepth: 1 + + how-to/run-container + + +++ + + Practical step-by-step guides for the more experienced user. + + .. grid-item-card:: :material-regular:`info;3em` + + .. toctree:: + :caption: Explanations + :maxdepth: 1 + + explanations/docs-structure + + +++ + + Explanations of how the library works and why it works that way. + + .. grid-item-card:: :material-regular:`menu_book;3em` + + .. toctree:: + :caption: Reference + :maxdepth: 1 + + reference/api + ../genindex + + +++ + + Technical reference material including APIs and release notes. diff --git a/docs/reference/api.rst b/docs/user/reference/api.rst similarity index 100% rename from docs/reference/api.rst rename to docs/user/reference/api.rst diff --git a/docs/tutorials/installation.rst b/docs/user/tutorials/installation.rst similarity index 56% rename from docs/tutorials/installation.rst rename to docs/user/tutorials/installation.rst index 399dc2c5..e90d3efb 100644 --- a/docs/tutorials/installation.rst +++ b/docs/user/tutorials/installation.rst @@ -7,7 +7,7 @@ Check your version of python You will need python 3.8 or later. You can check your version of python by typing into a terminal:: - python3 --version + $ python3 --version Create a virtual environment @@ -16,30 +16,23 @@ Create a virtual environment It is recommended that you install into a “virtual environment” so this installation will not interfere with any existing Python software:: - python3 -m venv /path/to/venv - source /path/to/venv/bin/activate + $ python3 -m venv /path/to/venv + $ source /path/to/venv/bin/activate Installing the library ---------------------- -You can now use ``pip`` to install the library:: +You can now use ``pip`` to install the library and its dependencies:: - python3 -m pip install python3-pip-skeleton + $ python3 -m pip install python3-pip-skeleton If you require a feature that is not currently released you can also install from github:: - python3 -m pip install git+https://github.com/DiamondLightSource/python3-pip-skeleton.git + $ python3 -m pip install git+https://github.com/DiamondLightSource/python3-pip-skeleton.git The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing:: - python3-pip-skeleton --version - -Running in a container ----------------------- - -To pull the container from github container registry and run:: - - docker run ghcr.io/DiamondLightSource/python3-pip-skeleton:main --version + $ python3-pip-skeleton --version diff --git a/setup.cfg b/setup.cfg index 5d9ac269..3daa1c9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ packages = find: # as per recommendation here https://hynek.me/articles/testing-packaging/ package_dir = =src - + setup_requires = setuptools_scm[toml]>=6.2 @@ -38,9 +38,11 @@ dev = mypy pipdeptree pre-commit + pydata-sphinx-theme pytest-cov setuptools_scm[toml]>=6.2 - sphinx-rtd-theme-github-versions + sphinx-copybutton + sphinx-design tox tox-direct types-mock diff --git a/src/python3_pip_skeleton/__main__.py b/src/python3_pip_skeleton/__main__.py index 1a97fb44..c680183b 100644 --- a/src/python3_pip_skeleton/__main__.py +++ b/src/python3_pip_skeleton/__main__.py @@ -11,6 +11,6 @@ def main(args=None): args = parser.parse_args(args) -# test with: pipenv run python -m python3_pip_skeleton +# test with: python -m python3_pip_skeleton if __name__ == "__main__": main() From 4d21bc31ee9a9b8120cb21dfb519c40a48aa3cd8 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 13 Sep 2022 08:51:26 +0000 Subject: [PATCH 09/50] Run sdist install in container workflow This saves another runner starting up just for this --- .github/workflows/code.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 0aae7fb5..13711d46 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -126,29 +126,20 @@ jobs: context: . labels: ${{ steps.meta.outputs.labels }} - - name: Check runtime + - name: Test cli works in runtime image run: for i in ${{ steps.meta.outputs.tags }}; do docker run ${i} --version; done + - name: Test cli works in sdist installed in local python + # ${GITHUB_REPOSITORY##*/} is the repo name without org + # Replace this with the cli command if different to the repo name + run: pip install dist/*.gz && ${GITHUB_REPOSITORY##*/} --version + - name: Upload build files uses: actions/upload-artifact@v3 with: name: dist path: dist - sdist: - needs: container - runs-on: ubuntu-latest - - steps: - - uses: actions/download-artifact@v3 - - - name: Install sdist in a venv and check cli works - # ${GITHUB_REPOSITORY##*/} is the repo name without org - # Replace this with the cli command if different to the repo name - run: | - pip install dist/*.gz - ${GITHUB_REPOSITORY##*/} --version - release: # upload to PyPI and make a release on every tag if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') From 52038569d52a5471e90f21b0d5ab5d685a18d19c Mon Sep 17 00:00:00 2001 From: Gary Yendell Date: Tue, 13 Sep 2022 09:44:58 +0100 Subject: [PATCH 10/50] Update CI badges --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 0450bd2b..a014631e 100644 --- a/README.rst +++ b/README.rst @@ -33,12 +33,12 @@ Or if it is a commandline tool then you might put some example commands here:: $ python -m python3_pip_skeleton --version -.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/workflows/Code%20CI/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions?query=workflow%3A%22Code+CI%22 +.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml/badge.svg?branch=main + :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml :alt: Code CI -.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/workflows/Docs%20CI/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions?query=workflow%3A%22Docs+CI%22 +.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml/badge.svg?branch=main + :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml :alt: Docs CI .. |coverage| image:: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton/branch/main/graph/badge.svg From d96f15a97efae989aabac10b5f815bd1e3ea143c Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 3 Oct 2022 21:12:20 +0100 Subject: [PATCH 11/50] Update code.yml Fixing a bug that occurs when releasing. (This is already fixed in the skeleton-cli project but failed to get copied to skeleton). --- .github/workflows/code.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 13711d46..89960f22 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -127,7 +127,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - name: Test cli works in runtime image - run: for i in ${{ steps.meta.outputs.tags }}; do docker run ${i} --version; done + # check that the first tag can run with --version parameter + run: docker run $(echo ${{ steps.meta.outputs.tags }} | sed -e 's/\s.*$//') --version - name: Test cli works in sdist installed in local python # ${GITHUB_REPOSITORY##*/} is the repo name without org From 68976f406d9c74bc2674897be2cefbd49dbaeba4 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 4 Oct 2022 11:03:29 +0100 Subject: [PATCH 12/50] Update .github/workflows/code.yml Co-authored-by: Tom C (DLS) <101418278+coretl@users.noreply.github.com> --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 89960f22..19773839 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -128,7 +128,7 @@ jobs: - name: Test cli works in runtime image # check that the first tag can run with --version parameter - run: docker run $(echo ${{ steps.meta.outputs.tags }} | sed -e 's/\s.*$//') --version + run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version - name: Test cli works in sdist installed in local python # ${GITHUB_REPOSITORY##*/} is the repo name without org From 92f10b4dfd56e314255147a989c0103b910f7291 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 14 Oct 2022 13:52:05 +0100 Subject: [PATCH 13/50] Fix make version switcher to use the right key Pin pydata-sphinx-theme to allow the build to complete https://github.com/pydata/pydata-sphinx-theme/issues/987 --- .github/pages/make_switcher.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 5c65d788..39c12772 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -59,7 +59,7 @@ def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[st def write_json(path: Path, repository: str, versions: str): org, repo_name = repository.split("/") struct = [ - dict(name=version, url=f"https://{org}.github.io/{repo_name}/{version}/") + dict(version=version, url=f"https://{org}.github.io/{repo_name}/{version}/") for version in versions ] text = json.dumps(struct, indent=2) diff --git a/setup.cfg b/setup.cfg index 3daa1c9b..857492eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ dev = mypy pipdeptree pre-commit - pydata-sphinx-theme + pydata-sphinx-theme < 0.10.1 pytest-cov setuptools_scm[toml]>=6.2 sphinx-copybutton From d35ffdbd28a1cba3fda62503151bcd1da82efbd4 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 14 Oct 2022 13:54:50 +0100 Subject: [PATCH 14/50] Add sphinx autobuild --- docs/conf.py | 1 - docs/developer/how-to/build-docs.rst | 18 ++++++++++++++++++ setup.cfg | 5 +++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0b6dd5bb..c4c2126a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,6 @@ git_branch = check_output("git branch --show-current".split(), cwd=root) version = git_branch.decode().strip() else: - branch = "main" version = release extensions = [ diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst index 9540de1c..79e3f780 100644 --- a/docs/developer/how-to/build-docs.rst +++ b/docs/developer/how-to/build-docs.rst @@ -17,4 +17,22 @@ locally with a web browse:: $ firefox build/html/index.html +Autobuild +--------- + +You can also run an autobuild process, which will watch your ``docs`` +directory for changes and rebuild whenever it sees changes, reloading any +browsers watching the pages:: + + $ tox -e docs autobuild + +You can view the pages at localhost:: + + $ firefox http://localhost:8000 + +If you are making changes to source code too, you can tell it to watch +changes in this directory too:: + + $ tox -e docs autobuild -- --watch src + .. _sphinx: https://www.sphinx-doc.org/ \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 857492eb..2e9dacb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ dev = pydata-sphinx-theme < 0.10.1 pytest-cov setuptools_scm[toml]>=6.2 + sphinx-autobuild sphinx-copybutton sphinx-design tox @@ -122,5 +123,5 @@ allowlist_externals = pre-commit commands = pre-commit run --all-files {posargs} [testenv:docs] -allowlist_externals = sphinx-build -commands = sphinx-build -EWT --keep-going docs build/html {posargs} +allowlist_externals = sphinx-build sphinx-autobuild +commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html From 7c11165980fe4ccc9260538c9490dfe0cdfe40b1 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Wed, 5 Oct 2022 15:21:53 +0100 Subject: [PATCH 15/50] Use PyPA action for PyPI Publish Use the official Python Packaging Authority (PyPA) Action to publish to PyPI --- .github/workflows/code.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 19773839..93dd6e37 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -162,7 +162,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ + uses: pypa/gh-action-pypi-publish@release/v1 + with: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: pipx run twine upload dist/*.whl dist/*.tar.gz From 1acb1d47463edeec386cdd82a549e652c57f9b46 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Wed, 5 Oct 2022 15:27:44 +0100 Subject: [PATCH 16/50] Fix password parameter --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 93dd6e37..690bb7cd 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -164,4 +164,4 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + password: ${{ secrets.PYPI_TOKEN }} From 5ff0a302f73174dec5eed0942ef1abd369264b5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:00:48 +0000 Subject: [PATCH 17/50] Bump black from 22.8.0 to 22.10.0 Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2e9dacb8..bd277c9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ setup_requires = [options.extras_require] # For development tests/docs dev = - black==22.8.0 + black==22.10.0 flake8-isort isort>5.0 mypy From 323424fb1d9d037e029ae45920770f656b2e991a Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 14 Oct 2022 15:31:24 +0100 Subject: [PATCH 18/50] Sanitize ref name for docs version Translate punctuation and unicode in branch names to _ --- .github/workflows/docs.yml | 8 +++++--- .github/workflows/docs_clean.yml | 13 ++++++++----- setup.cfg | 4 +++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a684d031..f0e7ebb6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,12 +41,14 @@ jobs: - name: Build docs run: tox -e docs + - name: Sanitize ref name for docs version + run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV + - name: Move to versioned directory - # e.g. main or 0.1.2 - run: mv build/html ".github/pages/${{ github.ref_name }}" + run: mv build/html .github/pages/$DOCS_VERSION - name: Write switcher.json - run: python .github/pages/make_switcher.py --add "${{ github.ref_name }}" ${{ github.repository }} .github/pages/switcher.json + run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json - name: Publish Docs to gh-pages if: github.event_name == 'push' diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index b80e4c22..d5425e42 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -24,17 +24,20 @@ jobs: - name: removing documentation for branch ${{ github.event.ref }} if: ${{ github.event_name != 'workflow_dispatch' }} - run: echo "remove_me=${{ github.event.ref }}" >> $GITHUB_ENV + run: echo "REF_NAME=${{ github.event.ref }}" >> $GITHUB_ENV - name: manually removing documentation version ${{ github.event.inputs.version }} if: ${{ github.event_name == 'workflow_dispatch' }} - run: echo "remove_me=${{ github.event.inputs.version }}" >> $GITHUB_ENV + run: echo "REF_NAME=${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Sanitize ref name for docs version + run: echo "DOCS_VERSION=${REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV - name: update index and push changes run: | - rm -r ${{ env.remove_me }} - python make_switcher.py --remove ${{ env.remove_me }} ${{ github.repository }} switcher.json + rm -r ${{ env.DOCS_VERSION }} + python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json git config --global user.name 'GitHub Actions Docs Cleanup CI' git config --global user.email 'GithubActionsCleanup@noreply.github.com' - git commit -am"removing redundant docs version ${{ env.remove_me }}" + git commit -am "Removing redundant docs version $DOCS_VERSION" git push diff --git a/setup.cfg b/setup.cfg index bd277c9a..87c5b596 100644 --- a/setup.cfg +++ b/setup.cfg @@ -123,5 +123,7 @@ allowlist_externals = pre-commit commands = pre-commit run --all-files {posargs} [testenv:docs] -allowlist_externals = sphinx-build sphinx-autobuild +allowlist_externals = + sphinx-build + sphinx-autobuild commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html From 964446a43cda17b235a5577dc586d89c0054f9af Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 18 Oct 2022 08:43:27 +0100 Subject: [PATCH 19/50] move requirements assests to lockfiles zip --- .containerignore | 7 ------- .dockerignore | 10 ++++++++++ .github/workflows/code.yml | 11 ++++++++++- .github/workflows/container_tests.sh | 3 ++- Dockerfile | 16 ++++++++++------ 5 files changed, 32 insertions(+), 15 deletions(-) delete mode 100644 .containerignore create mode 100644 .dockerignore diff --git a/.containerignore b/.containerignore deleted file mode 100644 index eb7d5ae1..00000000 --- a/.containerignore +++ /dev/null @@ -1,7 +0,0 @@ -Dockerfile -build/ -dist/ -.mypy_cache -.tox -.venv* -venv* diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4fb4c9ef --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +build/ +dist/ +.mypy_cache +.tox +.venv* +venv* +.devcontainer.json +.pre-commit-config.yaml +.vscode +README.rst diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 690bb7cd..d6322890 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -110,6 +110,7 @@ jobs: run: | docker run --name test build bash /project/.github/workflows/container_tests.sh docker cp test:/project/dist . + docker cp test:/project/lockfiles . docker cp test:/project/cov.xml . - name: Upload coverage to Codecov @@ -141,6 +142,12 @@ jobs: name: dist path: dist + - name: Upload lock files + uses: actions/upload-artifact@v3 + with: + name: lockfiles + path: lockfiles + release: # upload to PyPI and make a release on every tag if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') @@ -156,7 +163,9 @@ jobs: uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14 with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} - files: dist/* + files: | + dist/ + lockfiles/ generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/container_tests.sh b/.github/workflows/container_tests.sh index 5f921597..c36bbd9a 100644 --- a/.github/workflows/container_tests.sh +++ b/.github/workflows/container_tests.sh @@ -6,7 +6,8 @@ source /venv/bin/activate touch requirements_dev.txt pip install -r requirements_dev.txt -e .[dev] -pip freeze --exclude-editable > dist/requirements_dev.txt +mkdir -p lockfiles +pip freeze --exclude-editable > lockfiles/requirements_dev.txt pipdeptree diff --git a/Dockerfile b/Dockerfile index b8bfe727..5e04e438 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,29 +17,33 @@ RUN apt-get update && apt-get upgrade -y && \ && busybox --install COPY . /project +WORKDIR /project -RUN cd /project && \ - pip install --upgrade pip build && \ +# make the wheel outside of the venv so 'build' does not dirty requirements.txt +RUN pip install --upgrade pip build && \ export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ python -m build --sdist --wheel && \ touch requirements.txt +# set up a virtual environment and put it in PATH RUN python -m venv /venv ENV PATH=/venv/bin:$PATH ENV TOX_DIRECT=1 -RUN cd /project && \ - pip install --upgrade pip && \ +# install the wheel and generate the requirements file +RUN pip install --upgrade pip && \ pip install -r requirements.txt dist/*.whl && \ - pip freeze > dist/requirements.txt && \ + mkdir -p lockfiles && \ + pip freeze > lockfiles/requirements.txt && \ # we don't want to include our own wheel in requirements - remove with sed # and replace with a comment to avoid a zero length asset upload later - sed -i '/file:/s/^/# Requirements for /' dist/requirements.txt + sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt FROM python:3.10-slim as runtime # Add apt-get system dependecies for runtime here if needed +# copy the virtual environment from the build stage and put it in PATH COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH From b0426cef26f36c5cc379af879a8a40e1c558721f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 18 Oct 2022 12:22:16 +0100 Subject: [PATCH 20/50] fix .dockerignore, build options --- .dockerignore | 2 -- .gitignore | 2 ++ Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4fb4c9ef..e2ed7105 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,5 @@ dist/ .tox .venv* venv* -.devcontainer.json .pre-commit-config.yaml .vscode -README.rst diff --git a/.gitignore b/.gitignore index e0fba46a..9fbb6bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,6 @@ target/ .venv* venv* +# further build artifacts +lockfiles/ diff --git a/Dockerfile b/Dockerfile index 5e04e438..55020416 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /project # make the wheel outside of the venv so 'build' does not dirty requirements.txt RUN pip install --upgrade pip build && \ export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - python -m build --sdist --wheel && \ + python -m build && \ touch requirements.txt # set up a virtual environment and put it in PATH From a8d55dd788dc3b7ed7eba0278683475604d1d2d3 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 1 Nov 2022 08:25:50 +0000 Subject: [PATCH 21/50] add check for dirty repo when building wheel --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 55020416..c96bee05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ WORKDIR /project # make the wheel outside of the venv so 'build' does not dirty requirements.txt RUN pip install --upgrade pip build && \ export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + git diff && \ python -m build && \ touch requirements.txt From 081e205c8530f819115e9d8a99be7c3b4d112c45 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 1 Nov 2022 08:34:20 +0000 Subject: [PATCH 22/50] fix dockerignore to not dirty repo --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index e2ed7105..a6fab0ea 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,5 +4,3 @@ dist/ .tox .venv* venv* -.pre-commit-config.yaml -.vscode From 9da307f5c84f03f563ae16645d6f29cb6b9d00da Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 2 Nov 2022 07:59:43 +0000 Subject: [PATCH 23/50] fix Github Release assets spec --- .github/workflows/code.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d6322890..3aa05f1b 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -164,8 +164,8 @@ jobs: with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: | - dist/ - lockfiles/ + dist/* + lockfiles/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dd95a77887f0ae9cf7a8ee04749e85a2af848d3e Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 4 Nov 2022 12:29:53 +0000 Subject: [PATCH 24/50] Improve tox-direct handling - Environment variable no longer needs to be set - All commands run with tox-direct by default - All environment variables passed through --- Dockerfile | 1 - setup.cfg | 29 +++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index c96bee05..df6249c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,6 @@ RUN pip install --upgrade pip build && \ # set up a virtual environment and put it in PATH RUN python -m venv /venv ENV PATH=/venv/bin:$PATH -ENV TOX_DIRECT=1 # install the wheel and generate the requirements file RUN pip install --upgrade pip && \ diff --git a/setup.cfg b/setup.cfg index 87c5b596..abe2919d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -106,24 +106,21 @@ source = # NOTE that we pre-install all tools in the dev dependencies (including tox). # Hence the use of allowlist_externals instead of using the tox virtualenvs. # This ensures a match between developer time tools in the IDE and tox tools. -# Setting TOX_DIRECT=1 in the environment will make this even faster [tox:tox] skipsdist = True -[testenv:pytest] -allowlist_externals = pytest -commands = pytest {posargs} - -[testenv:mypy] -allowlist_externals = mypy -commands = mypy src tests {posargs} - -[testenv:pre-commit] -allowlist_externals = pre-commit -commands = pre-commit run --all-files {posargs} - -[testenv:docs] -allowlist_externals = +[testenv:{pre-commit,mypy,pytest,docs}] +# Don't create a virtualenv for the command, requires tox-direct plugin +direct = True +passenv = * +allowlist_externals = + pytest + pre-commit + mypy sphinx-build sphinx-autobuild -commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html +commands = + pytest: pytest {posargs} + mypy: mypy src tests {posargs} + pre-commit: pre-commit run --all-files {posargs} + docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html From 4ccb60169da316ebd2b80f99d0131ad7b0043790 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 8 Nov 2022 09:37:20 +0000 Subject: [PATCH 25/50] Rely on the container less - Moved wheel and sdist creation to the dist job - Rely on the test matrix to run tests - Simplified container build to make minimal for build and runtime and use wheel from 'dist': only publish to GHCR for tagged builds - Create separate requirements-*.txt for each of the test matrix - Fix actions-gh-pages version and don't run it for dependabot - Move Dockerfile to .devcontainer and use as context to improve build times - Other minor improvements and simplifications --- .devcontainer/Dockerfile | 37 +++++ .../devcontainer.json | 12 +- .dockerignore | 6 - .../actions/install_requirements/action.yml | 58 +++++++ .github/workflows/code.yml | 153 ++++++++++-------- .github/workflows/container_tests.sh | 14 -- .github/workflows/docs.yml | 30 ++-- .github/workflows/docs_clean.yml | 4 +- .github/workflows/linkcheck.yml | 19 +-- Dockerfile | 52 ------ setup.cfg | 3 +- 11 files changed, 212 insertions(+), 176 deletions(-) create mode 100644 .devcontainer/Dockerfile rename .devcontainer.json => .devcontainer/devcontainer.json (81%) delete mode 100644 .dockerignore create mode 100644 .github/actions/install_requirements/action.yml delete mode 100644 .github/workflows/container_tests.sh delete mode 100644 Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..b6b4bef9 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,37 @@ +# This file is for use as a devcontainer and a runtime container +# +# The devcontainer should use the build target and run as root with podman +# or docker with user namespaces. +# +FROM python:3.11 as build + +ARG PIP_OPTIONS + +# Add any system dependencies for the developer/build environment here e.g. +# RUN apt-get update && apt-get upgrade -y && \ +# apt-get install -y --no-install-recommends \ +# desired-packages \ +# && rm -rf /var/lib/apt/lists/* + +# set up a virtual environment and put it in PATH +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH + +# Copy any required context for the pip install over +COPY . /context +WORKDIR /context + +# install python package into /venv +RUN pip install ${PIP_OPTIONS} + +FROM python:3.11-slim as runtime + +# Add apt-get system dependecies for runtime here if needed + +# copy the virtual environment from the build stage and put it in PATH +COPY --from=build /venv/ /venv/ +ENV PATH=/venv/bin:$PATH + +# change this entrypoint if it is not the same as the repo +ENTRYPOINT ["python3-pip-skeleton"] +CMD ["--version"] diff --git a/.devcontainer.json b/.devcontainer/devcontainer.json similarity index 81% rename from .devcontainer.json rename to .devcontainer/devcontainer.json index d0921df6..1046ef9e 100644 --- a/.devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,16 +4,17 @@ "build": { "dockerfile": "Dockerfile", "target": "build", - "context": ".", - "args": {} + // Only upgrade pip, we will install the project below + "args": { + "PIP_OPTIONS": "--upgrade pip" + } }, "remoteEnv": { "DISPLAY": "${localEnv:DISPLAY}" }, // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/venv/bin/python", - "python.linting.enabled": true + "python.defaultInterpreterPath": "/venv/bin/python" }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ @@ -24,6 +25,7 @@ "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'", "runArgs": [ "--net=host", + "--security-opt=label=type:container_runtime_t", "-v=${localEnv:HOME}/.ssh:/root/.ssh", "-v=${localEnv:HOME}/.inputrc:/root/.inputrc" ], @@ -35,5 +37,5 @@ "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind", "workspaceFolder": "${localWorkspaceFolder}", // After the container is created, install the python project in editable form - "postCreateCommand": "pip install $([ -f requirements_dev.txt ] && echo -r requirements_dev.txt ) -e .[dev]" + "postCreateCommand": "pip install -e .[dev]" } \ No newline at end of file diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index a6fab0ea..00000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -build/ -dist/ -.mypy_cache -.tox -.venv* -venv* diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml new file mode 100644 index 00000000..25a146d1 --- /dev/null +++ b/.github/actions/install_requirements/action.yml @@ -0,0 +1,58 @@ +name: Install requirements +description: Run pip install with requirements and upload resulting requirements +inputs: + requirements_file: + description: Name of requirements file to use and upload + required: true + install_options: + description: Parameters to pass to pip install + required: true + python_version: + description: Python version to install + default: "3.x" + +runs: + using: composite + + steps: + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Pip install + run: | + touch ${{ inputs.requirements_file }} + # -c uses requirements.txt as constraints, see 'Validate requirements file' + pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }} + shell: bash + + - name: Create lockfile + run: | + mkdir -p lockfiles + pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} + # delete the self referencing line and make sure it isn't blank + sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + shell: bash + + - name: Upload lockfiles + uses: actions/upload-artifact@v3 + with: + name: lockfiles + path: lockfiles + + # This eliminates the class of problems where the requirements being given no + # longer match what the packages themselves dictate. E.g. In the rare instance + # where I install some-package which used to depend on vulnerable-dependency + # but now uses good-dependency (despite being nominally the same version) + # pip will install both if given a requirements file with -r + - name: If requirements file exists, check it matches pip installed packages + run: | + if [ -s ${{ inputs.requirements_file }} ]; then + if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then + echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive" + exit 1 + fi + fi + shell: bash + diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 3aa05f1b..200b07b6 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -4,8 +4,11 @@ on: push: pull_request: schedule: - # Run every Monday at 8am to check latest versions of dependencies + # Run weekly to check latest versions of dependencies - cron: "0 8 * * WED" +env: + # The target python version, which must match the Dockerfile version + CONTAINER_PYTHON: "3.11" jobs: lint: @@ -17,16 +20,14 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Setup python - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: "3.10" + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Lint - run: | - touch requirements_dev.txt requirements.txt - pip install -r requirements.txt -r requirements_dev.txt -e .[dev] - tox -e pre-commit,mypy + run: tox -e pre-commit,mypy test: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository @@ -34,7 +35,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.8", "3.9", "3.10"] + python: ["3.9", "3.10", "3.11"] + install: ["-e .[dev]"] + # Make one version be non-editable to test both paths of version code + include: + - os: "ubuntu-latest" + python: "3.8" + install: ".[dev]" runs-on: ${{ matrix.os }} env: @@ -45,18 +52,21 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: + # Need this to get version number from last tag fetch-depth: 0 - - name: Setup python ${{ matrix.python }} - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: ${{ matrix.python }} + python_version: ${{ matrix.python }} + requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt + install_options: ${{ matrix.install }} - - name: Install with latest dependencies - run: pip install .[dev] + - name: List dependency tree + run: pipdeptree - name: Run tests - run: pytest tests + run: pytest - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -64,9 +74,46 @@ jobs: name: ${{ matrix.python }}/${{ matrix.os }} files: cov.xml - container: + dist: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Build sdist and wheel + run: | + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + pipx run build + + - name: Upload sdist and wheel as artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Check for packaging errors + run: pipx run twine check dist/* + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + python_version: ${{env.CONTAINER_PYTHON}} + requirements_file: requirements.txt + install_options: dist/*.whl + + - name: Test module --version works using the installed wheel + # If more than one module in src/ replace with module name to test + run: python -m $(ls src | head -1) --version + + container: + needs: [lint, dist, test] runs-on: ubuntu-latest + permissions: contents: read packages: write @@ -74,8 +121,15 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + + # image names must be all lower case + - name: Generate image repo name + run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV + + - name: Download wheel and lockfiles + uses: actions/download-artifact@v3 with: - fetch-depth: 0 + path: .devcontainer - name: Log in to GitHub Docker Registry if: github.event_name != 'pull_request' @@ -89,74 +143,45 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/${{ github.repository }} + images: ${{ env.IMAGE_REPOSITORY }} tags: | - type=ref,event=branch type=ref,event=tag + type=raw,value=latest - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 - - name: Build developer image for testing - uses: docker/build-push-action@v3 - with: - tags: build:latest - context: . - target: build - load: true - - - name: Run tests in the container locked with requirements_dev.txt - run: | - docker run --name test build bash /project/.github/workflows/container_tests.sh - docker cp test:/project/dist . - docker cp test:/project/lockfiles . - docker cp test:/project/cov.xml . - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - name: 3.10-locked/ubuntu-latest - files: cov.xml - - name: Build runtime image uses: docker/build-push-action@v3 with: - push: ${{ github.event_name != 'pull_request' }} + build-args: | + PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} + load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} tags: ${{ steps.meta.outputs.tags }} - context: . - labels: ${{ steps.meta.outputs.labels }} + context: .devcontainer + # If you have a long docker build, uncomment the following to turn on caching + # For short build times this makes it a little slower + #cache-from: type=gha + #cache-to: type=gha,mode=max - name: Test cli works in runtime image - # check that the first tag can run with --version parameter - run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version - - - name: Test cli works in sdist installed in local python - # ${GITHUB_REPOSITORY##*/} is the repo name without org - # Replace this with the cli command if different to the repo name - run: pip install dist/*.gz && ${GITHUB_REPOSITORY##*/} --version - - - name: Upload build files - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist - - - name: Upload lock files - uses: actions/upload-artifact@v3 - with: - name: lockfiles - path: lockfiles + run: docker run ${{ env.IMAGE_REPOSITORY }} --version release: # upload to PyPI and make a release on every tag - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - needs: container + needs: [lint, dist, test] + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 + - name: Fixup blank lockfiles + # Github release artifacts can't be blank + run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done + - name: Github Release # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions diff --git a/.github/workflows/container_tests.sh b/.github/workflows/container_tests.sh deleted file mode 100644 index c36bbd9a..00000000 --- a/.github/workflows/container_tests.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -x - -cd /project -source /venv/bin/activate - -touch requirements_dev.txt -pip install -r requirements_dev.txt -e .[dev] -mkdir -p lockfiles -pip freeze --exclude-editable > lockfiles/requirements_dev.txt - -pipdeptree - -pytest tests diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f0e7ebb6..94fa2151 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,11 +7,6 @@ on: jobs: docs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - fail-fast: false - matrix: - python: ["3.10"] - runs-on: ubuntu-latest steps: @@ -19,24 +14,21 @@ jobs: if: startsWith(github.ref, 'refs/tags') run: sleep 60 - - name: Install python version - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v3 with: - python-version: ${{ matrix.python }} + # Need this to get version number from last tag + fetch-depth: 0 - - name: Install Packages + - name: Install system packages # Can delete this if you don't use graphviz in your docs run: sudo apt-get install graphviz - - name: checkout - uses: actions/checkout@v3 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - fetch-depth: 0 - - - name: Install dependencies - run: | - touch requirements_dev.txt - pip install -r requirements_dev.txt -e .[dev] + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Build docs run: tox -e docs @@ -51,10 +43,10 @@ jobs: run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json - name: Publish Docs to gh-pages - if: github.event_name == 'push' + if: github.event_name == 'push' && github.actor != 'dependabot[bot]' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 + uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index d5425e42..a67e1881 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: checkout + - name: Checkout uses: actions/checkout@v3 with: ref: gh-pages @@ -35,7 +35,7 @@ jobs: - name: update index and push changes run: | - rm -r ${{ env.DOCS_VERSION }} + rm -r $DOCS_VERSION python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json git config --global user.name 'GitHub Actions Docs Cleanup CI' git config --global user.email 'GithubActionsCleanup@noreply.github.com' diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index e6838560..42d199c4 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,8 +1,9 @@ name: Link Check on: + workflow_dispatch: schedule: - # Run every Monday at 8am to check URL links still resolve + # Run weekly to check URL links still resolve - cron: "0 8 * * WED" jobs: @@ -17,18 +18,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install python version - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: ${{ matrix.python }} - - - name: Install dependencies - run: | - touch requirements_dev.txt - pip install -r requirements_dev.txt -e .[dev] + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Check links - run: tox -e docs -- -b linkcheck + run: tox -e docs build -- -b linkcheck diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index df6249c9..00000000 --- a/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# This file is for use as a devcontainer and a runtime container -# -# The devcontainer should use the build target and run as root with podman -# or docker with user namespaces. -# -FROM python:3.10 as build - -# Add any system dependencies for the developer/build environment here -RUN apt-get update && apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - build-essential \ - busybox \ - git \ - net-tools \ - vim \ - && rm -rf /var/lib/apt/lists/* \ - && busybox --install - -COPY . /project -WORKDIR /project - -# make the wheel outside of the venv so 'build' does not dirty requirements.txt -RUN pip install --upgrade pip build && \ - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - git diff && \ - python -m build && \ - touch requirements.txt - -# set up a virtual environment and put it in PATH -RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH - -# install the wheel and generate the requirements file -RUN pip install --upgrade pip && \ - pip install -r requirements.txt dist/*.whl && \ - mkdir -p lockfiles && \ - pip freeze > lockfiles/requirements.txt && \ - # we don't want to include our own wheel in requirements - remove with sed - # and replace with a comment to avoid a zero length asset upload later - sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt - -FROM python:3.10-slim as runtime - -# Add apt-get system dependecies for runtime here if needed - -# copy the virtual environment from the build stage and put it in PATH -COPY --from=build /venv/ /venv/ -ENV PATH=/venv/bin:$PATH - -# change this entrypoint if it is not the same as the repo -ENTRYPOINT ["python3-pip-skeleton"] -CMD ["--version"] diff --git a/setup.cfg b/setup.cfg index abe2919d..ba377343 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 [options] python_requires = >=3.8 @@ -50,8 +51,6 @@ dev = [options.packages.find] where = src -# Don't include our tests directory in the distribution -exclude = tests # Specify any package data to be included in the wheel below. # [options.package_data] From fa99c26a72861597d419620622d859df9d65a722 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Mon, 14 Nov 2022 13:08:36 +0000 Subject: [PATCH 26/50] Mount ssh & inputrc in mounts list --- .devcontainer/devcontainer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1046ef9e..7c7d6da1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,11 +25,11 @@ "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'", "runArgs": [ "--net=host", - "--security-opt=label=type:container_runtime_t", - "-v=${localEnv:HOME}/.ssh:/root/.ssh", - "-v=${localEnv:HOME}/.inputrc:/root/.inputrc" + "--security-opt=label=type:container_runtime_t" ], "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind", + "source=${localEnv:HOME}/.inputrc,target=/root/.inputrc,type=bind", // map in home directory - not strictly necessary but useful "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached" ], From c5f583f6cce94d157d45a3288e62332ace63097a Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 11 Nov 2022 15:44:26 +0000 Subject: [PATCH 27/50] Moved config to pyproject.toml --- .vscode/extensions.json | 3 +- pyproject.toml | 108 ++++++++++++++++++++++++++ setup.cfg | 125 ------------------------------ tests/test_boilerplate_removed.py | 14 ++-- 4 files changed, 116 insertions(+), 134 deletions(-) delete mode 100644 setup.cfg diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 041f8944..fe4a5809 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,6 +3,7 @@ "ms-vscode-remote.remote-containers", "ms-python.vscode-pylance", "ms-python.python", - "ryanluker.vscode-coverage-gutters" + "ryanluker.vscode-coverage-gutters", + "tamasfe.even-better-toml" ] } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1b8c998a..1c710455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,5 +2,113 @@ requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel"] build-backend = "setuptools.build_meta" +[project] +name = "python3-pip-skeleton" +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +description = "One line description of your module" +dependencies = [] # Add project dependencies here, e.g. ["click", "numpy"] +dynamic = ["version"] +license.file = "LICENSE" +readme = "README.rst" +requires-python = ">=3.8" + +[project.optional-dependencies] +dev = [ + "black", + "isort", + "mypy", + "flake8", + "flake8-isort", + "Flake8-pyproject", + "pipdeptree", + "pre-commit", + "pydata-sphinx-theme", + "pytest-cov", + "setuptools_scm[toml]>=6.2", + "sphinx-autobuild", + "sphinx-copybutton", + "sphinx-design", + "tox", + "tox-direct", + "types-mock", +] + +[project.scripts] +python3-pip-skeleton = "python3_pip_skeleton.__main__:main" + +[project.urls] +GitHub = "https://github.com/DiamondLightSource/python3-pip-skeleton" + +[[project.authors]] # Further authors may be added by duplicating this section +email = "email@address.com" +name = "Firstname Lastname" + + [tool.setuptools_scm] write_to = "src/python3_pip_skeleton/_version.py" + +[tool.mypy] +ignore_missing_imports = true # Ignore missing stubs in imported modules + +[tool.isort] +float_to_top = true +profile = "black" + +[tool.flake8] +extend-ignore = [ + "E203", # See https://github.com/PyCQA/pycodestyle/issues/373 + "F811", # support typing.overload decorator + "F722", # allow Annotated[typ, some_func("some string")] +] +max-line-length = 88 # Respect black's line length (default 88), +exclude = [".tox", "venv"] + + +[tool.pytest.ini_options] +# Run pytest with all our checkers, and don't spam us with massive tracebacks on error +addopts = """ + --tb=native -vv --doctest-modules --doctest-glob="*.rst" + --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml + """ +# https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings +filterwarnings = "error" +# Doctest python code in docs, python code in src docstrings, test functions in tests +testpaths = "docs src tests" + +[tool.coverage.run] +data_file = "/tmp/python3_pip_skeleton.coverage" + +[tool.coverage.paths] +# Tests are run from installed location, map back to the src directory +source = ["src", "**/site-packages/"] + +# tox must currently be configured via an embedded ini string +# See: https://github.com/tox-dev/tox/issues/999 +[tool.tox] +legacy_tox_ini = """ +[tox] +skipsdist=True + +[testenv:{pre-commit,mypy,pytest,docs}] +# Don't create a virtualenv for the command, requires tox-direct plugin +direct = True +passenv = * +allowlist_externals = + pytest + pre-commit + mypy + sphinx-build + sphinx-autobuild +commands = + pytest: pytest {posargs} + mypy: mypy src tests {posargs} + pre-commit: pre-commit run --all-files {posargs} + docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html +""" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ba377343..00000000 --- a/setup.cfg +++ /dev/null @@ -1,125 +0,0 @@ -[metadata] -name = python3-pip-skeleton -description = One line description of your module -url = https://github.com/DiamondLightSource/python3-pip-skeleton -author = Firstname Lastname -author_email = email@address.com -license = Apache License 2.0 -long_description = file: README.rst -long_description_content_type = text/x-rst -classifiers = - License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - -[options] -python_requires = >=3.8 -packages = find: -# =src is interpreted as {"": "src"} -# as per recommendation here https://hynek.me/articles/testing-packaging/ -package_dir = - =src - -setup_requires = - setuptools_scm[toml]>=6.2 - -# Specify any package dependencies below. -# install_requires = -# numpy -# scipy - -[options.extras_require] -# For development tests/docs -dev = - black==22.10.0 - flake8-isort - isort>5.0 - mypy - pipdeptree - pre-commit - pydata-sphinx-theme < 0.10.1 - pytest-cov - setuptools_scm[toml]>=6.2 - sphinx-autobuild - sphinx-copybutton - sphinx-design - tox - tox-direct - types-mock - -[options.packages.find] -where = src - -# Specify any package data to be included in the wheel below. -# [options.package_data] -# python3_pip_skeleton = -# subpackage/*.yaml - -[options.entry_points] -# Include a command line script -console_scripts = - python3-pip-skeleton = python3_pip_skeleton.__main__:main - -[mypy] -# Ignore missing stubs for modules we use -ignore_missing_imports = True - -[isort] -profile=black -float_to_top=true - -[flake8] -# Make flake8 respect black's line length (default 88), -max-line-length = 88 -extend-ignore = - E203, # See https://github.com/PyCQA/pycodestyle/issues/373 - F811, # support typing.overload decorator - F722, # allow Annotated[typ, some_func("some string")] -exclude = - .tox - venv - -[tool:pytest] -# Run pytest with all our checkers, and don't spam us with massive tracebacks on error -addopts = - --tb=native -vv --doctest-modules --doctest-glob="*.rst" - --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml -# https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings -filterwarnings = error -# Doctest python code in docs, python code in src docstrings, test functions in tests -testpaths = - docs src tests - -[coverage:run] -data_file = /tmp/python3_pip_skeleton.coverage - -[coverage:paths] -# Tests are run from installed location, map back to the src directory -source = - src - **/site-packages/ - -# Use tox to provide parallel linting and testing -# NOTE that we pre-install all tools in the dev dependencies (including tox). -# Hence the use of allowlist_externals instead of using the tox virtualenvs. -# This ensures a match between developer time tools in the IDE and tox tools. -[tox:tox] -skipsdist = True - -[testenv:{pre-commit,mypy,pytest,docs}] -# Don't create a virtualenv for the command, requires tox-direct plugin -direct = True -passenv = * -allowlist_externals = - pytest - pre-commit - mypy - sphinx-build - sphinx-autobuild -commands = - pytest: pytest {posargs} - mypy: mypy src tests {posargs} - pre-commit: pre-commit run --all-files {posargs} - docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html diff --git a/tests/test_boilerplate_removed.py b/tests/test_boilerplate_removed.py index f5204fa9..b823c53b 100644 --- a/tests/test_boilerplate_removed.py +++ b/tests/test_boilerplate_removed.py @@ -2,7 +2,7 @@ This file checks that all the example boilerplate text has been removed. It can be deleted when all the contained tests pass """ -import configparser +from importlib.metadata import metadata from pathlib import Path ROOT = Path(__file__).parent.parent @@ -24,14 +24,12 @@ def assert_not_contains_text(path: str, text: str, explanation: str): skeleton_check(text in contents, f"Please change ./{path} {explanation}") -# setup.cfg -def test_module_description(): - conf = configparser.ConfigParser() - conf.read("setup.cfg") - description = conf["metadata"]["description"] +# pyproject.toml +def test_module_summary(): + summary = metadata("python3-pip-skeleton")["summary"] skeleton_check( - "One line description of your module" in description, - "Please change description in ./setup.cfg " + "One line description of your module" in summary, + "Please change project.description in ./pyproject.toml " "to be a one line description of your module", ) From a08124fd6d4950732098ab3af7e3c781690ec7c2 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 17 Nov 2022 05:20:59 +0000 Subject: [PATCH 28/50] add version label to container registry push --- .github/workflows/code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 200b07b6..a3a1c487 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -160,6 +160,7 @@ jobs: push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} context: .devcontainer # If you have a long docker build, uncomment the following to turn on caching # For short build times this makes it a little slower From d97fd677d6e7e666aaf6c8d1e079d338e75e00db Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 11 Nov 2022 16:06:36 +0000 Subject: [PATCH 29/50] Use importlib.metadata to get package version --- pyproject.toml | 1 - src/python3_pip_skeleton/__init__.py | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c710455..0bcc80db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dev = [ "pre-commit", "pydata-sphinx-theme", "pytest-cov", - "setuptools_scm[toml]>=6.2", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", diff --git a/src/python3_pip_skeleton/__init__.py b/src/python3_pip_skeleton/__init__.py index 0fe6655f..ef94dff1 100644 --- a/src/python3_pip_skeleton/__init__.py +++ b/src/python3_pip_skeleton/__init__.py @@ -1,12 +1,6 @@ -try: - # Use live version from git - from setuptools_scm import get_version +from importlib.metadata import version - # Warning: If the install is nested to the same depth, this will always succeed - __version__ = get_version(root="../../", relative_to=__file__) - del get_version -except (ImportError, LookupError): - # Use installed version - from ._version import __version__ +__version__ = version("python3-pip-skeleton") +del version __all__ = ["__version__"] From 7383739e7f5ae5cc4309eded64774c7c4bf58c1d Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Thu, 17 Nov 2022 12:02:43 +0000 Subject: [PATCH 30/50] Make twine check strict --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index a3a1c487..5ff24420 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -97,7 +97,7 @@ jobs: path: dist - name: Check for packaging errors - run: pipx run twine check dist/* + run: pipx run twine check --strict dist/* - name: Install python packages uses: ./.github/actions/install_requirements From 15a1d441d72fc2132bae767d0a947dccfe22ba33 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Thu, 24 Nov 2022 14:44:08 +0000 Subject: [PATCH 31/50] Don't check switcher if not published - pydata-sphinx-theme 0.11 started checking switcher - this meant you couldn't bootstrap a gh-pages build - pydata-sphinx-theme 0.12 put in an option not to check - but we want checking if the file exists - so only check if we can get the json file - and suggest user turns pages on if we can't --- docs/conf.py | 17 ++++++++++++++++- pyproject.toml | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c4c2126a..6bd4d230 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,9 +4,12 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +import sys from pathlib import Path from subprocess import check_output +import requests + import python3_pip_skeleton # -- General configuration ------------------------------------------------ @@ -126,6 +129,17 @@ html_theme = "pydata_sphinx_theme" github_repo = project github_user = "DiamondLightSource" +switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" +# Don't check switcher if it doesn't exist, but warn in a non-failing way +check_switcher = requests.get(switcher_json).ok +if not check_switcher: + print( + "*** Can't read version switcher, is GitHub pages enabled? \n" + " Once Docs CI job has successfully run once, set the " + "Github pages source branch to be 'gh-pages' at:\n" + f" https://github.com/{github_user}/{github_repo}/settings/pages", + file=sys.stderr, + ) # Theme options for pydata_sphinx_theme html_theme_options = dict( @@ -142,9 +156,10 @@ ) ], switcher=dict( - json_url=f"https://{github_user}.github.io/{github_repo}/switcher.json", + json_url=switcher_json, version_match=version, ), + check_switcher=check_switcher, navbar_end=["theme-switcher", "icon-links", "version-switcher"], external_links=[ dict( diff --git a/pyproject.toml b/pyproject.toml index 0bcc80db..5e06a4f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dev = [ "Flake8-pyproject", "pipdeptree", "pre-commit", - "pydata-sphinx-theme", + "pydata-sphinx-theme>=0.12", "pytest-cov", "sphinx-autobuild", "sphinx-copybutton", From 6f0604bf457831bc4824de810461ccbd19fd854c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 25 Nov 2022 13:43:30 +0000 Subject: [PATCH 32/50] Don't use flake8==6 until plugins catch up https://github.com/john-hen/Flake8-pyproject/issues/12 --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5e06a4f0..bf1da9b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ dev = [ "black", "isort", "mypy", - "flake8", + # https://github.com/john-hen/Flake8-pyproject/issues/12 + "flake8<6", "flake8-isort", "Flake8-pyproject", "pipdeptree", From 7a688851d9c4491d420b92c624da16408cabddf1 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 28 Nov 2022 11:45:00 +0000 Subject: [PATCH 33/50] Remove flake8 constraint Also shrink dep list where intermediate modules are covered by others --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf1da9b4..0397a22d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,10 +22,7 @@ requires-python = ">=3.8" [project.optional-dependencies] dev = [ "black", - "isort", "mypy", - # https://github.com/john-hen/Flake8-pyproject/issues/12 - "flake8<6", "flake8-isort", "Flake8-pyproject", "pipdeptree", @@ -35,7 +32,6 @@ dev = [ "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", - "tox", "tox-direct", "types-mock", ] From d4633ab1f059eebbd76f08ddfec979b36f81396c Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 18 Nov 2022 17:26:38 +0000 Subject: [PATCH 34/50] Add agreed upon extensions to customizations --- .devcontainer/devcontainer.json | 16 +++++++++++----- .vscode/extensions.json | 6 +++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7c7d6da1..7f5c7e3b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,11 +16,17 @@ "settings": { "python.defaultInterpreterPath": "/venv/bin/python" }, - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance" - ], + "customizations": { + "vscode": { + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "ryanluker.vscode-coverage-gutters" + ] + } + }, // Make sure the files we are mapping into the container exist on the host "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'", "runArgs": [ diff --git a/.vscode/extensions.json b/.vscode/extensions.json index fe4a5809..81922991 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,9 +1,9 @@ { "recommendations": [ "ms-vscode-remote.remote-containers", - "ms-python.vscode-pylance", "ms-python.python", - "ryanluker.vscode-coverage-gutters", - "tamasfe.even-better-toml" + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "ryanluker.vscode-coverage-gutters" ] } \ No newline at end of file From f916925ab167a0aa79d7c34e63ebe68d06631aab Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 25 Nov 2022 16:29:06 +0000 Subject: [PATCH 35/50] Add common-utils to dev container features --- .devcontainer/devcontainer.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7f5c7e3b..6c707ab0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,6 +12,13 @@ "remoteEnv": { "DISPLAY": "${localEnv:DISPLAY}" }, + // Add the URLs of features you want added when the container is built. + "features": { + "ghcr.io/devcontainers/features/common-utils:1": { + "username": "none", + "upgradePackages": false + } + }, // Set *default* container specific settings.json values on container create. "settings": { "python.defaultInterpreterPath": "/venv/bin/python" From 4981421ca3aab9b9b39ae8dcace211ab519d3187 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 2 Dec 2022 15:48:42 +0000 Subject: [PATCH 36/50] Link to condatiners.dev for devcontainer spec --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6c707ab0..f5e71fe9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -// For format details, see https://aka.ms/devcontainer.json +// For format details, see https://containers.dev/implementors/json_reference/ { "name": "Python 3 Developer Container", "build": { From 7300883bd8a3cb5ba2e338c728d320cbc5681922 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Wed, 7 Dec 2022 13:36:36 +0000 Subject: [PATCH 37/50] Remove unused matrix from linkcheck CI --- .github/workflows/linkcheck.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 42d199c4..02d8df4c 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -8,11 +8,6 @@ on: jobs: docs: - strategy: - fail-fast: false - matrix: - python: ["3.10"] - runs-on: ubuntu-latest steps: From f67ddd4e6c86db2521c6855c8b6d4bf0aab20089 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Wed, 30 Nov 2022 16:44:32 +0000 Subject: [PATCH 38/50] Add publish to anaconda step --- .github/workflows/code.yml | 38 +++++++++++++++++++++++++++++++++++++- README.rst | 7 ++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 5ff24420..7c671aae 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -170,11 +170,42 @@ jobs: - name: Test cli works in runtime image run: docker run ${{ env.IMAGE_REPOSITORY }} --version + conda: + needs: [dist] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v3 + + - name: Create Conda recipe + run: | + mkdir -p conda/recipe + pipx run grayskull pypi -o conda/recipe dist/*.tar.gz + + - name: Install conda-build + run: conda install -y conda-build + + - name: Build Conda distribution + run: | + mkdir -p conda/build + conda build --output-folder conda/build conda/recipe/*/meta.yaml + + - name: Upload conda build + uses: actions/upload-artifact@v3 + with: + name: conda_build + path: | + conda/build/ + !conda/build/*/.cache + release: # upload to PyPI and make a release on every tag - needs: [lint, dist, test] + needs: [lint, dist, test, conda] if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} runs-on: ubuntu-latest + env: + HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} + HAS_ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN != '' }} steps: - uses: actions/download-artifact@v3 @@ -197,6 +228,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish to PyPI + if: ${{ env.HAS_PYPI_TOKEN }} uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} + + - name: Publish to Anaconda + if: ${{ env.HAS_ANACONDA_TOKEN }} + run: pipx run --spec anaconda-client anaconda --token ${{ secrets.ANACONDA_TOKEN }} upload conda_build/noarch/*.tar.bz2 diff --git a/README.rst b/README.rst index a014631e..78f7f140 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ python3-pip-skeleton =========================== -|code_ci| |docs_ci| |coverage| |pypi_version| |license| +|code_ci| |docs_ci| |coverage| |pypi_version| |anaconda_version| |license| .. note:: @@ -14,6 +14,7 @@ how it does it, and why people should use it. ============== ============================================================== PyPI ``pip install python3-pip-skeleton`` +Conda ``conda install -c DiamondLightSource python3-pip-skeleton`` Source code https://github.com/DiamondLightSource/python3-pip-skeleton Documentation https://DiamondLightSource.github.io/python3-pip-skeleton Releases https://github.com/DiamondLightSource/python3-pip-skeleton/releases @@ -49,6 +50,10 @@ Or if it is a commandline tool then you might put some example commands here:: :target: https://pypi.org/project/python3-pip-skeleton :alt: Latest PyPI version +.. |anaconda_version| image:: https://anaconda.org/DiamondLightSource/python3-pip-skeleton/badges/version.svg + :target: https://anaconda.org/DiamondLightSource/python3-pip-skeleton + :alt: Latest Anaconda version + .. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg :target: https://opensource.org/licenses/Apache-2.0 :alt: Apache License From ea18cfd21ad278e4abd9223a9a76ef635f9efabd Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 10:26:16 +0000 Subject: [PATCH 39/50] Rename python3-pip-skeleton -> scanspec --- .devcontainer/Dockerfile | 2 +- .github/CONTRIBUTING.rst | 6 +-- README.rst | 42 +++++++++---------- docs/conf.py | 8 ++-- docs/developer/how-to/make-release.rst | 2 +- docs/developer/tutorials/dev-install.rst | 6 +-- docs/index.rst | 4 +- docs/user/how-to/run-container.rst | 6 +-- docs/user/reference/api.rst | 8 ++-- docs/user/tutorials/installation.rst | 6 +-- pyproject.toml | 16 +++---- .../__init__.py | 2 +- .../__main__.py | 2 +- tests/test_cli.py | 4 +- 14 files changed, 57 insertions(+), 57 deletions(-) rename src/{python3_pip_skeleton => scanspec}/__init__.py (63%) rename src/{python3_pip_skeleton => scanspec}/__main__.py (86%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b6b4bef9..0266d11e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,5 +33,5 @@ COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH # change this entrypoint if it is not the same as the repo -ENTRYPOINT ["python3-pip-skeleton"] +ENTRYPOINT ["scanspec"] CMD ["--version"] diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 19ab494f..a2fa3ae6 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -7,7 +7,7 @@ filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project. -.. _GitHub: https://github.com/DiamondLightSource/python3-pip-skeleton/issues +.. _GitHub: https://github.com/dls-controls/scanspec/issues Issue or Discussion? -------------------- @@ -16,7 +16,7 @@ Github also offers discussions_ as a place to ask questions and share ideas. If your issue is open ended and it is not obvious when it can be "closed", please raise it as a discussion instead. -.. _discussions: https://github.com/DiamondLightSource/python3-pip-skeleton/discussions +.. _discussions: https://github.com/dls-controls/scanspec/discussions Code coverage ------------- @@ -32,4 +32,4 @@ The `Developer Guide`_ contains information on setting up a development environment, running the tests and what standards the code and documentation should follow. -.. _Developer Guide: https://diamondlightsource.github.io/python3-pip-skeleton/main/developer/how-to/contribute.html +.. _Developer Guide: https://diamondlightsource.github.io/scanspec/main/developer/how-to/contribute.html diff --git a/README.rst b/README.rst index 78f7f140..37f45fb6 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -python3-pip-skeleton +scanspec =========================== |code_ci| |docs_ci| |coverage| |pypi_version| |anaconda_version| |license| @@ -7,17 +7,17 @@ python3-pip-skeleton This project contains template code only. For documentation on how to adopt this skeleton project see - https://DiamondLightSource.github.io/python3-pip-skeleton-cli + https://dls-controls.github.io/scanspec-cli This is where you should write a short paragraph that describes what your module does, how it does it, and why people should use it. ============== ============================================================== -PyPI ``pip install python3-pip-skeleton`` -Conda ``conda install -c DiamondLightSource python3-pip-skeleton`` -Source code https://github.com/DiamondLightSource/python3-pip-skeleton -Documentation https://DiamondLightSource.github.io/python3-pip-skeleton -Releases https://github.com/DiamondLightSource/python3-pip-skeleton/releases +PyPI ``pip install scanspec`` +Conda ``conda install -c dls-controls scanspec`` +Source code https://github.com/dls-controls/scanspec +Documentation https://dls-controls.github.io/scanspec +Releases https://github.com/dls-controls/scanspec/releases ============== ============================================================== This is where you should put some images or code snippets that illustrate @@ -26,32 +26,32 @@ introductory code here: .. code-block:: python - from python3_pip_skeleton import __version__ + from scanspec import __version__ - print(f"Hello python3_pip_skeleton {__version__}") + print(f"Hello scanspec {__version__}") Or if it is a commandline tool then you might put some example commands here:: - $ python -m python3_pip_skeleton --version + $ python -m scanspec --version -.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/code.yml +.. |code_ci| image:: https://github.com/dls-controls/scanspec/actions/workflows/code.yml/badge.svg?branch=main + :target: https://github.com/dls-controls/scanspec/actions/workflows/code.yml :alt: Code CI -.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml/badge.svg?branch=main - :target: https://github.com/DiamondLightSource/python3-pip-skeleton/actions/workflows/docs.yml +.. |docs_ci| image:: https://github.com/dls-controls/scanspec/actions/workflows/docs.yml/badge.svg?branch=main + :target: https://github.com/dls-controls/scanspec/actions/workflows/docs.yml :alt: Docs CI -.. |coverage| image:: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton/branch/main/graph/badge.svg - :target: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton +.. |coverage| image:: https://codecov.io/gh/dls-controls/scanspec/branch/main/graph/badge.svg + :target: https://codecov.io/gh/dls-controls/scanspec :alt: Test Coverage -.. |pypi_version| image:: https://img.shields.io/pypi/v/python3-pip-skeleton.svg - :target: https://pypi.org/project/python3-pip-skeleton +.. |pypi_version| image:: https://img.shields.io/pypi/v/scanspec.svg + :target: https://pypi.org/project/scanspec :alt: Latest PyPI version -.. |anaconda_version| image:: https://anaconda.org/DiamondLightSource/python3-pip-skeleton/badges/version.svg - :target: https://anaconda.org/DiamondLightSource/python3-pip-skeleton +.. |anaconda_version| image:: https://anaconda.org/dls-controls/scanspec/badges/version.svg + :target: https://anaconda.org/dls-controls/scanspec :alt: Latest Anaconda version .. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg @@ -62,4 +62,4 @@ Or if it is a commandline tool then you might put some example commands here:: Anything below this line is used when viewing README.rst and will be replaced when included in index.rst -See https://DiamondLightSource.github.io/python3-pip-skeleton for more detailed documentation. +See https://dls-controls.github.io/scanspec for more detailed documentation. diff --git a/docs/conf.py b/docs/conf.py index 6bd4d230..c854bd40 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,15 +10,15 @@ import requests -import python3_pip_skeleton +import scanspec # -- General configuration ------------------------------------------------ # General information about the project. -project = "python3-pip-skeleton" +project = "scanspec" # The full version, including alpha/beta/rc tags. -release = python3_pip_skeleton.__version__ +release = scanspec.__version__ # The short X.Y version. if "+" in release: @@ -128,7 +128,7 @@ # html_theme = "pydata_sphinx_theme" github_repo = project -github_user = "DiamondLightSource" +github_user = "dls-controls" switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" # Don't check switcher if it doesn't exist, but warn in a non-failing way check_switcher = requests.get(switcher_json).ok diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst index 747e44a2..d7614e74 100644 --- a/docs/developer/how-to/make-release.rst +++ b/docs/developer/how-to/make-release.rst @@ -13,4 +13,4 @@ To make a new release, please follow this checklist: Note that tagging and pushing to the main branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/DiamondLightSource/python3-pip-skeleton/releases \ No newline at end of file +.. _release: https://github.com/dls-controls/scanspec/releases \ No newline at end of file diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index c2632683..e8f385d8 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -10,7 +10,7 @@ Clone the repository First clone the repository locally using `Git `_:: - $ git clone git://github.com/DiamondLightSource/python3-pip-skeleton.git + $ git clone git://github.com/dls-controls/scanspec.git Install dependencies -------------------- @@ -25,7 +25,7 @@ requires python 3.8 or later) or to run in a container under `VSCode .. code:: - $ cd python3-pip-skeleton + $ cd scanspec $ python3 -m venv venv $ source venv/bin/activate $ pip install -e .[dev] @@ -34,7 +34,7 @@ requires python 3.8 or later) or to run in a container under `VSCode .. code:: - $ vscode python3-pip-skeleton + $ vscode scanspec # Click on 'Reopen in Container' when prompted # Open a new terminal diff --git a/docs/index.rst b/docs/index.rst index df33c8e6..61032e34 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,13 +14,13 @@ The documentation is split into 2 sections: :link: user/index :link-type: doc - The User Guide contains documentation on how to install and use python3-pip-skeleton. + The User Guide contains documentation on how to install and use scanspec. .. grid-item-card:: :material-regular:`code;4em` :link: developer/index :link-type: doc - The Developer Guide contains documentation on how to develop and contribute changes back to python3-pip-skeleton. + The Developer Guide contains documentation on how to develop and contribute changes back to scanspec. .. toctree:: :hidden: diff --git a/docs/user/how-to/run-container.rst b/docs/user/how-to/run-container.rst index 84f857af..67ab907e 100644 --- a/docs/user/how-to/run-container.rst +++ b/docs/user/how-to/run-container.rst @@ -1,15 +1,15 @@ Run in a container ================== -Pre-built containers with python3-pip-skeleton and its dependencies already +Pre-built containers with scanspec and its dependencies already installed are available on `Github Container Registry -`_. +`_. Starting the container ---------------------- To pull the container from github container registry and run:: - $ docker run ghcr.io/DiamondLightSource/python3-pip-skeleton:main --version + $ docker run ghcr.io/dls-controls/scanspec:main --version To get a released version, use a numbered release instead of ``main``. diff --git a/docs/user/reference/api.rst b/docs/user/reference/api.rst index 8544e172..91e2f472 100644 --- a/docs/user/reference/api.rst +++ b/docs/user/reference/api.rst @@ -1,14 +1,14 @@ API === -.. automodule:: python3_pip_skeleton +.. automodule:: scanspec - ``python3_pip_skeleton`` + ``scanspec`` ----------------------------------- -This is the internal API reference for python3_pip_skeleton +This is the internal API reference for scanspec -.. data:: python3_pip_skeleton.__version__ +.. data:: scanspec.__version__ :type: str Version number as calculated by https://github.com/pypa/setuptools_scm diff --git a/docs/user/tutorials/installation.rst b/docs/user/tutorials/installation.rst index e90d3efb..0a0a6f02 100644 --- a/docs/user/tutorials/installation.rst +++ b/docs/user/tutorials/installation.rst @@ -25,14 +25,14 @@ Installing the library You can now use ``pip`` to install the library and its dependencies:: - $ python3 -m pip install python3-pip-skeleton + $ python3 -m pip install scanspec If you require a feature that is not currently released you can also install from github:: - $ python3 -m pip install git+https://github.com/DiamondLightSource/python3-pip-skeleton.git + $ python3 -m pip install git+https://github.com/dls-controls/scanspec.git The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing:: - $ python3-pip-skeleton --version + $ scanspec --version diff --git a/pyproject.toml b/pyproject.toml index 0397a22d..386c936c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=64", "setuptools_scm[toml]>=6.2", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "python3-pip-skeleton" +name = "scanspec" classifiers = [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", @@ -37,18 +37,18 @@ dev = [ ] [project.scripts] -python3-pip-skeleton = "python3_pip_skeleton.__main__:main" +scanspec = "scanspec.__main__:main" [project.urls] -GitHub = "https://github.com/DiamondLightSource/python3-pip-skeleton" +GitHub = "https://github.com/dls-controls/scanspec" [[project.authors]] # Further authors may be added by duplicating this section -email = "email@address.com" -name = "Firstname Lastname" +email = "tom.cobb@diamond.ac.uk" +name = "Tom Cobb" [tool.setuptools_scm] -write_to = "src/python3_pip_skeleton/_version.py" +write_to = "src/scanspec/_version.py" [tool.mypy] ignore_missing_imports = true # Ignore missing stubs in imported modules @@ -71,7 +71,7 @@ exclude = [".tox", "venv"] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error addopts = """ --tb=native -vv --doctest-modules --doctest-glob="*.rst" - --cov=python3_pip_skeleton --cov-report term --cov-report xml:cov.xml + --cov=scanspec --cov-report term --cov-report xml:cov.xml """ # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings filterwarnings = "error" @@ -79,7 +79,7 @@ filterwarnings = "error" testpaths = "docs src tests" [tool.coverage.run] -data_file = "/tmp/python3_pip_skeleton.coverage" +data_file = "/tmp/scanspec.coverage" [tool.coverage.paths] # Tests are run from installed location, map back to the src directory diff --git a/src/python3_pip_skeleton/__init__.py b/src/scanspec/__init__.py similarity index 63% rename from src/python3_pip_skeleton/__init__.py rename to src/scanspec/__init__.py index ef94dff1..44045b1f 100644 --- a/src/python3_pip_skeleton/__init__.py +++ b/src/scanspec/__init__.py @@ -1,6 +1,6 @@ from importlib.metadata import version -__version__ = version("python3-pip-skeleton") +__version__ = version("scanspec") del version __all__ = ["__version__"] diff --git a/src/python3_pip_skeleton/__main__.py b/src/scanspec/__main__.py similarity index 86% rename from src/python3_pip_skeleton/__main__.py rename to src/scanspec/__main__.py index c680183b..2328bfe5 100644 --- a/src/python3_pip_skeleton/__main__.py +++ b/src/scanspec/__main__.py @@ -11,6 +11,6 @@ def main(args=None): args = parser.parse_args(args) -# test with: python -m python3_pip_skeleton +# test with: python -m scanspec if __name__ == "__main__": main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 2ff648c0..97851192 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,9 @@ import subprocess import sys -from python3_pip_skeleton import __version__ +from scanspec import __version__ def test_cli_version(): - cmd = [sys.executable, "-m", "python3_pip_skeleton", "--version"] + cmd = [sys.executable, "-m", "scanspec", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ From 473ede66ccc2bbb3cd6390f9c6d89a54014b3c30 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 12:22:20 +0000 Subject: [PATCH 40/50] Fix tests with new dependencies --- README.rst | 6 ------ pyproject.toml | 10 ++++------ src/scanspec/cli.py | 2 +- src/scanspec/sphinxext.py | 2 +- tests/test_boilerplate_removed.py | 2 +- tests/test_cli.py | 5 ++++- 6 files changed, 11 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 240d1a13..5df70873 100644 --- a/README.rst +++ b/README.rst @@ -3,12 +3,6 @@ scanspec |code_ci| |docs_ci| |coverage| |pypi_version| |anaconda_version| |license| -.. note:: - - This project contains template code only. For documentation on how to - adopt this skeleton project see - https://dls-controls.github.io/scanspec-cli - Specify step and flyscan Paths using combinations of: - Specs like Line or Spiral diff --git a/pyproject.toml b/pyproject.toml index 4ab32bc5..4b77843f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "numpy>=1.19.3", "click", "apischema>=0.17.2", - "typing_extensions" + "typing_extensions", ] dynamic = ["version"] license.file = "LICENSE" @@ -29,13 +29,10 @@ requires-python = ">=3.8" plotting = [ # make sure a python 3.9 compatible scipy and matplotlib are selected "scipy>=1.5.4", - "matplotlib>=3.2.2" + "matplotlib>=3.2.2", ] # GraphQL service support -graphql = [ - "graphql-server[aiohttp]==3.0.0b5", - "aiohttp_cors" -] +graphql = ["graphql-server[aiohttp]==3.0.0b5", "aiohttp_cors"] # For development tests/docs dev = [ "black==22.3.0", @@ -46,6 +43,7 @@ dev = [ "pre-commit", "pydata-sphinx-theme>=0.12", "pytest-cov", + "pydocstyle", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", diff --git a/src/scanspec/cli.py b/src/scanspec/cli.py index fb7b534e..75a3cf22 100644 --- a/src/scanspec/cli.py +++ b/src/scanspec/cli.py @@ -16,7 +16,7 @@ ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"], case_sensitive=False ), ) -@click.version_option() +@click.version_option(prog_name="scanspec") @click.pass_context def cli(ctx, log_level: str): """Top level scanspec command line interface.""" diff --git a/src/scanspec/sphinxext.py b/src/scanspec/sphinxext.py index 8e5edbe0..ecde40d9 100644 --- a/src/scanspec/sphinxext.py +++ b/src/scanspec/sphinxext.py @@ -2,7 +2,7 @@ from matplotlib.sphinxext import plot_directive -from ._version_git import __version__ +from . import __version__ @contextmanager diff --git a/tests/test_boilerplate_removed.py b/tests/test_boilerplate_removed.py index b823c53b..22949dcc 100644 --- a/tests/test_boilerplate_removed.py +++ b/tests/test_boilerplate_removed.py @@ -26,7 +26,7 @@ def assert_not_contains_text(path: str, text: str, explanation: str): # pyproject.toml def test_module_summary(): - summary = metadata("python3-pip-skeleton")["summary"] + summary = metadata("scanspec")["summary"] skeleton_check( "One line description of your module" in summary, "Please change project.description in ./pyproject.toml " diff --git a/tests/test_cli.py b/tests/test_cli.py index 30dd396b..64a81018 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -230,4 +230,7 @@ def test_schema() -> None: def test_cli_version(): cmd = [sys.executable, "-m", "scanspec", "--version"] - assert subprocess.check_output(cmd).decode().strip() == __version__ + assert ( + subprocess.check_output(cmd).decode().strip() + == f"scanspec, version {__version__}" + ) From 7171265de5a968d38a8a83b6fe08617fb2a4e627 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 12:32:18 +0000 Subject: [PATCH 41/50] Downgrade mypy and fix issues --- pyproject.toml | 2 +- src/scanspec/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4b77843f..16f34a7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ graphql = ["graphql-server[aiohttp]==3.0.0b5", "aiohttp_cors"] # For development tests/docs dev = [ "black==22.3.0", - "mypy", + "mypy<=0.930", "flake8-isort", "Flake8-pyproject", "pipdeptree", diff --git a/src/scanspec/core.py b/src/scanspec/core.py index 9774d7b0..6f1e872b 100644 --- a/src/scanspec/core.py +++ b/src/scanspec/core.py @@ -618,7 +618,7 @@ def axes(self) -> List[Axis]: def __len__(self) -> int: """The number of dictionaries that will be produced if iterated over.""" - return np.product([len(frames) for frames in self.stack]) + return int(np.product([len(frames) for frames in self.stack])) def __iter__(self) -> Iterator[Dict[Axis, float]]: """Yield {axis: midpoint} for each frame in the scan.""" From dc5e1b806e1f6a106f9fe71fcedeb2b827c58c66 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 12:40:22 +0000 Subject: [PATCH 42/50] Upgrade mypy again --- pyproject.toml | 2 +- src/scanspec/core.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 16f34a7b..ff322e71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ graphql = ["graphql-server[aiohttp]==3.0.0b5", "aiohttp_cors"] # For development tests/docs dev = [ "black==22.3.0", - "mypy<=0.930", + "mypy<=0.982", "flake8-isort", "Flake8-pyproject", "pipdeptree", diff --git a/src/scanspec/core.py b/src/scanspec/core.py index 6f1e872b..e041055e 100644 --- a/src/scanspec/core.py +++ b/src/scanspec/core.py @@ -419,6 +419,7 @@ def extract(self, indices: np.ndarray, calculate_gap=True) -> Frames[Axis]: length = len(self) backwards = (indices // length) % 2 snake_indices = np.where(backwards, (length - 1) - indices, indices) % length + cls: Type[Frames[Any]] if not calculate_gap: cls = Frames gap = self.gap[np.where(backwards, length - indices, indices) % length] From 8410ff2aa32f40908ceaaccfe3d6d30e7b21abae Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 12:43:27 +0000 Subject: [PATCH 43/50] Remove wheel check --- .github/workflows/code.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index ebbe1b07..3fddd931 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -34,11 +34,6 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Install wheel in a venv and check cli works - # ${GITHUB_REPOSITORY##*/} is the repo name without org - # Replace this with the cli command if different to the repo name - run: pipx run --python $(which python${{ matrix.python }}) --spec dist/*.whl ${GITHUB_REPOSITORY##*/} --version - test: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository strategy: From ec9fee80be8e66e283118b9f39cef16110f797d3 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 13:58:27 +0000 Subject: [PATCH 44/50] Fix docs build following migration --- docs/conf.py | 2 ++ docs/genindex.rst | 5 ---- docs/how-to/contribute.rst | 1 - docs/index.rst | 50 ++++++++++++++++++++++----------- docs/reference/contributing.rst | 1 + 5 files changed, 37 insertions(+), 22 deletions(-) delete mode 100644 docs/genindex.rst delete mode 100644 docs/how-to/contribute.rst create mode 100644 docs/reference/contributing.rst diff --git a/docs/conf.py b/docs/conf.py index ce442c29..15fe8905 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,6 +65,8 @@ nitpick_ignore = [ ("py:func", "int"), ("py:class", "Axis"), + ("py:class", "~Axis"), + ("py:class", "scanspec.core.Axis"), ("py:class", "AxesPoints"), ("py:class", "np.ndarray"), ("py:class", "NoneType"), diff --git a/docs/genindex.rst b/docs/genindex.rst deleted file mode 100644 index 93eb8b29..00000000 --- a/docs/genindex.rst +++ /dev/null @@ -1,5 +0,0 @@ -API Index -========= - -.. - https://stackoverflow.com/a/42310803 diff --git a/docs/how-to/contribute.rst b/docs/how-to/contribute.rst deleted file mode 100644 index 65b992f0..00000000 --- a/docs/how-to/contribute.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../../.github/CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 362452e9..545dc316 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,3 @@ -:html_theme.sidebar_secondary.remove: - .. include:: ../README.rst :end-before: when included in index.rst @@ -8,24 +6,44 @@ How the documentation is structured ----------------------------------- -The documentation is split into 2 sections: +Documentation is split into four categories, also accessible from links in the +side-bar. + +.. rst-class:: columns + +`tutorials` +~~~~~~~~~~~ + +.. include:: tutorials.rst + :start-after: ========= + +.. rst-class:: columns + +`how-to` +~~~~~~~~ + +.. include:: how-to.rst + :start-after: ============= + +.. rst-class:: columns + +`explanations` +~~~~~~~~~~~~~~ -.. grid:: 2 +.. include:: explanations.rst + :start-after: ============ - .. grid-item-card:: :material-regular:`person;4em` - :link: user/index - :link-type: doc +.. rst-class:: columns - The User Guide contains documentation on how to install and use scanspec. +`reference` +~~~~~~~~~~~ - .. grid-item-card:: :material-regular:`code;4em` - :link: developer/index - :link-type: doc +.. include:: reference.rst + :start-after: ========= - The Developer Guide contains documentation on how to develop and contribute changes back to scanspec. +.. rst-class:: endcolumns -.. toctree:: - :hidden: +About the documentation +~~~~~~~~~~~~~~~~~~~~~~~ - user/index - developer/index +`Why is the documentation structured this way? `_ diff --git a/docs/reference/contributing.rst b/docs/reference/contributing.rst new file mode 100644 index 00000000..65b72c8e --- /dev/null +++ b/docs/reference/contributing.rst @@ -0,0 +1 @@ +.. include:: ../../.github/CONTRIBUTING.rst From 1281a3775da9addf7dcf18b080850f23a3502286 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 13:59:28 +0000 Subject: [PATCH 45/50] Install extra depdencies for tests --- .github/workflows/code.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 3fddd931..643c5e15 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -41,12 +41,12 @@ jobs: matrix: os: ["ubuntu-latest"] # can add windows-latest, macos-latest python: ["3.9", "3.10", "3.11"] - install: ["-e .[dev]"] + install: ["-e .[dev,graphql,plotting]"] # Make one version be non-editable to test both paths of version code include: - os: "ubuntu-latest" python: "3.8" - install: ".[dev]" + install: ".[dev,graphql,plotting]" runs-on: ${{ matrix.os }} env: From 6dd2ede92bbd3ba31163732a42c822b1b9581c99 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 14:01:39 +0000 Subject: [PATCH 46/50] Install all extras in docs CI --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 94fa2151..dc8136ca 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/install_requirements with: requirements_file: requirements-dev-3.x.txt - install_options: -e .[dev] + install_options: -e .[dev,graphql,plotting] - name: Build docs run: tox -e docs From 656c797cf4980f3d8d6d6638d5c21c22bc596065 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 14:02:46 +0000 Subject: [PATCH 47/50] Remove conda from CI --- .github/workflows/code.yml | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 643c5e15..6b97ca74 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -175,42 +175,13 @@ jobs: - name: Test cli works in runtime image run: docker run ${{ env.IMAGE_REPOSITORY }} --version - conda: - needs: [dist] - runs-on: ubuntu-latest - - steps: - - uses: actions/download-artifact@v3 - - - name: Create Conda recipe - run: | - mkdir -p conda/recipe - pipx run grayskull pypi -o conda/recipe dist/*.tar.gz - - - name: Install conda-build - run: conda install -y conda-build - - - name: Build Conda distribution - run: | - mkdir -p conda/build - conda build --output-folder conda/build conda/recipe/*/meta.yaml - - - name: Upload conda build - uses: actions/upload-artifact@v3 - with: - name: conda_build - path: | - conda/build/ - !conda/build/*/.cache - release: # upload to PyPI and make a release on every tag - needs: [lint, dist, test, conda] + needs: [lint, dist, test] if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} runs-on: ubuntu-latest env: HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} - HAS_ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN != '' }} steps: - uses: actions/download-artifact@v3 @@ -237,7 +208,3 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} - - - name: Publish to Anaconda - if: ${{ env.HAS_ANACONDA_TOKEN }} - run: pipx run --spec anaconda-client anaconda --token ${{ secrets.ANACONDA_TOKEN }} upload conda_build/noarch/*.tar.bz2 From 2721d7d9d69c362e375d995b597dd072c8f0e679 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 14:28:53 +0000 Subject: [PATCH 48/50] Remove esbonio config --- .vscode/settings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 02676642..b73c801b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,5 @@ "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.organizeImports": true - }, - "esbonio.sphinx.confDir": "" + } } \ No newline at end of file From 576d923e8f9150a02d35851befde432192f3db92 Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 15:01:43 +0000 Subject: [PATCH 49/50] Downgrade matplotlib --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff322e71..8dc29894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ requires-python = ">=3.8" plotting = [ # make sure a python 3.9 compatible scipy and matplotlib are selected "scipy>=1.5.4", - "matplotlib>=3.2.2", + "matplotlib>=3.2.2,<3.6", ] # GraphQL service support graphql = ["graphql-server[aiohttp]==3.0.0b5", "aiohttp_cors"] From 987588f2d40207b81a83004a9f5f266899098bca Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Mon, 19 Dec 2022 15:23:29 +0000 Subject: [PATCH 50/50] Fix toctree in docs --- docs/index.rst | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 545dc316..2f712348 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,41 +9,37 @@ How the documentation is structured Documentation is split into four categories, also accessible from links in the side-bar. -.. rst-class:: columns +.. grid:: 4 -`tutorials` -~~~~~~~~~~~ + .. grid-item-card:: :material-regular:`person;4em` + :link: tutorials + :link-type: doc -.. include:: tutorials.rst - :start-after: ========= + Tutorials -.. rst-class:: columns + .. grid-item-card:: :material-regular:`code;4em` + :link: how-to + :link-type: doc -`how-to` -~~~~~~~~ + How-To -.. include:: how-to.rst - :start-after: ============= + .. grid-item-card:: :material-regular:`person;4em` + :link: explanations + :link-type: doc -.. rst-class:: columns + Explanations -`explanations` -~~~~~~~~~~~~~~ + .. grid-item-card:: :material-regular:`code;4em` + :link: reference + :link-type: doc -.. include:: explanations.rst - :start-after: ============ + Reference -.. rst-class:: columns -`reference` -~~~~~~~~~~~ +.. toctree:: + :hidden: -.. include:: reference.rst - :start-after: ========= - -.. rst-class:: endcolumns - -About the documentation -~~~~~~~~~~~~~~~~~~~~~~~ - -`Why is the documentation structured this way? `_ + tutorials + how-to + explanations + reference