From fb525addf9a197ceb977967cdd886904b6906996 Mon Sep 17 00:00:00 2001 From: Lucas Cimon <925560+Lucas-C@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:12:34 +0100 Subject: [PATCH 1/2] CI: restructuring jobs, hardening base image, and introducing guarddog (#1332) --- .../continuous-integration-workflow.yml | 180 ++++++++++++------ CHANGELOG.md | 4 +- README.md | 2 +- docs/Development.md | 3 +- docs/requirements.txt | 1 + test/image/image_types/test_insert_images.py | 4 + 6 files changed, 128 insertions(+), 66 deletions(-) diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index a95a94a9a..7a602c798 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -9,6 +9,66 @@ on: # cf. https://github.community/t/how-to-trigger-an-action-on-push-or-pull-r - master jobs: + check-reference-pdf-files: + runs-on: ubuntu-latest + steps: + - name: Checkout πŸ›ŽοΈ + uses: actions/checkout@v4 + - name: Set up Python 3.13 πŸ”§ + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Install system dependencies βš™οΈ + run: sudo apt-get update --allow-releaseinfo-change && sudo apt-get install qpdf + - name: Check all PDF reference files used in tests β˜‘ + run: | + # Using qpdf + find . -name '*.pdf' | xargs -n 1 sh -c 'qpdf --check --password=fpdf2 $0 || exit 255' + export PYTHONPATH=$PWD + # Using VeraPDF: + scripts/install-verapdf.sh + scripts/verapdf.py --process-all-test-pdf-files + scripts/verapdf.py --print-aggregated-report + # Using Datalogics PDF Checker: + scripts/install-pdfchecker.sh + scripts/pdfchecker.py --process-all-test-pdf-files + scripts/pdfchecker.py --print-aggregated-report + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout πŸ›ŽοΈ + uses: actions/checkout@v4 + - name: Set up Python 3.13 πŸ”§ + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Install Python dependencies βš™οΈ + run: | + python -m pip install --upgrade pip setuptools wheel + pip install --upgrade . -r test/linters-requirements.txt -r test/requirements.txt + - name: Run linters πŸ”Ž + run: | + black --check . + pylint fpdf test tutorial/tuto*.py + bandit -c .banditrc.yml -r contributors/ fpdf/ tutorial/ + semgrep scan --config auto --error --strict --exclude-rule=python.lang.security.insecure-hash-function.insecure-hash-function fpdf + - name: Scan project with grype πŸ”Ž + uses: anchore/scan-action@v3 + with: + path: "." + fail-build: true + - name: Scan project dependencies with guarddog 🐢 + run: | + pip install guarddog + # Scanning direct dependencies: + guarddog pypi scan defusedxml + guarddog pypi scan Pillow + guarddog pypi scan fonttools + # Scanning dev dependencies: + guarddog pypi verify contributors/requirements.txt + guarddog pypi verify docs/requirements.txt + guarddog pypi verify test/linters-requirements.txt + guarddog pypi verify test/requirements.txt test: strategy: matrix: @@ -17,51 +77,24 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Checkout πŸ›ŽοΈ - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} πŸ”§ - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install system dependencies βš™οΈ if: matrix.platform == 'ubuntu-latest' - run: sudo apt-get update --allow-releaseinfo-change && sudo apt-get install ghostscript libjpeg-dev + # Ghostscript is needed for test/table/test_table_extraction.py + run: sudo apt-get update --allow-releaseinfo-change && sudo apt-get install ghostscript - name: Install qpdf βš™οΈ - if: matrix.platform == 'ubuntu-latest' && matrix.python-version != '3.9' # We run the unit tests WITHOUT qpdf for a single parallel execution / Python version: + if: matrix.platform == 'ubuntu-latest' && matrix.python-version != '3.13' run: sudo apt-get update --allow-releaseinfo-change && sudo apt-get install qpdf - name: Install Python dependencies βš™οΈ run: | python -m pip install --upgrade pip setuptools wheel - pip install --upgrade . -r test/requirements.txt -r docs/requirements.txt -r contributors/requirements.txt - - name: Statically checking code πŸ”Ž - if: matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' - run: | - pip install --upgrade . -r test/linters-requirements.txt - black --check . - pylint fpdf test tutorial/tuto*.py - bandit -c .banditrc.yml -r contributors/ fpdf/ tutorial/ - semgrep scan --config auto --error --strict --exclude-rule=python.lang.security.insecure-hash-function.insecure-hash-function fpdf - - name: Scan current project - if: matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' - uses: anchore/scan-action@v3 - with: - path: "." - fail-build: true - - name: Checking all PDF samples β˜‘ - if: matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' - run: | - # Using qpdf - find . -name '*.pdf' | xargs -n 1 sh -c 'qpdf --check --password=fpdf2 $0 || exit 255' - export PYTHONPATH=$PWD - # Using VeraPDF: - scripts/install-verapdf.sh - scripts/verapdf.py --process-all-test-pdf-files - scripts/verapdf.py --print-aggregated-report - # Using Datalogics PDF Checker: - scripts/install-pdfchecker.sh - scripts/pdfchecker.py --process-all-test-pdf-files - scripts/pdfchecker.py --print-aggregated-report - - name: Running tests β˜‘ + pip install --upgrade . -r test/requirements.txt + - name: Run tests β˜‘ env: CHECK_EXEC_TIME: ${{ matrix.python-version == '3.9' && 'test-enabled' || '' }} CHECK_RSS_MEMORY: ${{ matrix.python-version == '3.13' && 'test-enabled' || '' }} @@ -70,24 +103,41 @@ jobs: grep -IRF generate=True test/ && exit 1 # Executing all tests: pytest -vv --trace-memory-usage - - name: Running tests with the minimal versions of fpdf2 direct dependencies β˜‘ - if: matrix.python-version == '3.8' && matrix.platform == 'ubuntu-latest' + - name: Upload coverage report to codecov.io β˜‘ + # We only upload coverage ONCE, for a single parallel execution / Python version: + if: matrix.platform == 'ubuntu-latest' && matrix.python-version == '3.13' + run: bash <(curl -s https://codecov.io/bash) + - name: Run tests with the minimal versions of fpdf2 direct dependencies β˜‘ + if: matrix.platform == 'ubuntu-latest' && matrix.python-version == '3.8' run: | - # We ensure that those minimal versions remain compatible: + # Ensuring that those minimal versions remain compatible: sed -i '/install_requires/,/\n/s/>=/==/' setup.cfg pip install . - # Targetting only a subset of tests because: A) it's faster and B) some tests are dependant on a specific version of fonttools or Pillow + # Targeting only a subset of tests because: A) it's faster and B) some tests are dependant on a specific version of fonttools or Pillow pytest -vv test/barcodes test/drawing test/errors test/image/test_load_image.py test/metadata test/shapes test/signing test/text_region test/utils - - name: Uploading coverage report to codecov.io β˜‘ - if: matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' - run: bash <(curl -s https://codecov.io/bash) - - name: Generating HTML documentation πŸ—οΈ - # As build_contributors_html_page.py can hang due to GitHub rate-limiting, - # we only execute this on master for now. And it should always be executed for one Python version only. - if: github.ref == 'refs/heads/master' && matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' - env: - # Needed by contributors/build_contributors_html_page.py: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + doc: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: block + allowed-endpoints: + github.com:443 + *.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 + - name: Checkout πŸ›ŽοΈ + uses: actions/checkout@v4 + - name: Set up Python 3.13 πŸ”§ + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Install Python dependencies βš™οΈ + run: | + python -m pip install --upgrade pip setuptools wheel + pip install --upgrade -r docs/requirements.txt -r contributors/requirements.txt + - name: Generate HTML documentation πŸ—οΈ run: | mkdir -p public/ # Setting PDF manual version: @@ -96,12 +146,17 @@ jobs: mkdocs build pdoc --html -o public/ fpdf --config "git_link_template='https://github.com/py-pdf/fpdf2/blob/{commit}/{path}#L{start_line}-L{end_line}'" scripts/add_pdoc_to_search_index.py + - name: Build contributors map πŸ—ΊοΈ + # As build_contributors_html_page.py can hang due to GitHub rate-limiting, we only execute this on master for now + if: github.ref == 'refs/heads/master' + env: + # Needed by contributors/build_contributors_html_page.py: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | cd contributors/ && PYTHONUNBUFFERED=1 ./build_contributors_html_page.py py-pdf/fpdf2 cp -t ../public/ contributors.html contributors-map-small.png - name: Deploy documentation πŸš€ - # GitHub Pages deployment should not be done for all Python versions, - # otherwise commits will conflict on the gh-pages branch: - if: github.ref == 'refs/heads/master' && matrix.python-version == '3.13' && matrix.platform == 'ubuntu-latest' + if: github.ref == 'refs/heads/master' uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -116,13 +171,22 @@ jobs: # Trusted publishing configured there: https://pypi.org/manage/project/fpdf2/settings/publishing/ id-token: write steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + egress-policy: block + allowed-endpoints: + github.com:443 + *.githubusercontent.com:443 + pypi.org:443 + files.pythonhosted.org:443 - name: Checkout πŸ›ŽοΈ - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.13 πŸ”§ - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.13' - - name: Building distributions for Pypi πŸ—οΈ + - name: Build distributions for Pypi πŸ—οΈ id: build run: | echo Versions already released on Pypi: $(curl -Ls 'https://pypi.org/pypi/fpdf2/json' | jq -r '.releases|keys[]') @@ -136,11 +200,3 @@ jobs: # Doc: https://github.com/marketplace/actions/pypi-publish with: print-hash: true - - name: Generate & release sigstore signatures πŸ”‘ - if: steps.build.outputs.publish == 'yes' - uses: sigstore/gh-action-sigstore-python@v3.0.0 - # Doc: https://github.com/marketplace/actions/gh-action-sigstore-python - with: - inputs: ./dist/*.tar.gz ./dist/*.whl - # For this setting to work, this pipeline should be triggered by a "release" event: - release-signing-artifacts: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e69f5b50..97c73d0e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,11 +28,11 @@ This can also be enabled programmatically with `warnings.simplefilter('default', * documentation on how to: [render spreadsheets as PDF tables](https://py-pdf.github.io/fpdf2/RenderingSpreadsheetsAsPDFTables.html) * support for passing `Align` values (along with string values like `'C'`, `'L'`, `'R'`) in `l_margin` of `TextStyle` to horizontally align text - [issue #1282](https://github.com/py-pdf/fpdf2/issues/1282) ### Fixed -* support for `align=` in [`FPDF.table()`](https://py-pdf.github.io/fpdf2/Tables.html#setting-table-column-widths). Due to this correction, tables are now properly horizontally aligned on the page by default. This was always specified in the documentation, but was not in effect until now. You can revert to have left-aligned tables by passing `align="LEFT"` to `FPDF.table()`. +* support for `align=` in [`FPDF.table()`](https://py-pdf.github.io/fpdf2/Tables.html#setting-table-column-widths). Due to this correction, tables are now properly horizontally aligned on the page by default. This was always specified in the documentation, but was not in effect until now. You can revert to have left-aligned tables by passing `align="LEFT"` to `FPDF.table()`. - [issue #1306](https://github.com/py-pdf/fpdf2/issues/1306) * `FPDF.set_text_shaping(False)` was broken since version 2.7.8 and is now working properly - [issue #1287](https://github.com/py-pdf/fpdf2/issues/1287) * fixed bug where cells with `rowspan`, `colspan` > 1 and null text were not displayed properly - [issue #1293](https://github.com/py-pdf/fpdf2/issues/1293) * `CreationDate` metadata used a wrong timezone offset for UTC - [issue #1261](https://github.com/py-pdf/fpdf2/issues/1261) -* [`insert_toc_placeholder()`](https://py-pdf.github.io/fpdf2/fpdf/fpdf.html#fpdf.fpdf.FPDF.insert_toc_placeholder) did not properly the page orientation, which caused a bug when the last page of the document was in a different orientation - [issue #1213](https://github.com/py-pdf/fpdf2/issues/1213) +* [`insert_toc_placeholder()`](https://py-pdf.github.io/fpdf2/fpdf/fpdf.html#fpdf.fpdf.FPDF.insert_toc_placeholder) did not properly the page orientation, which caused a bug when the last page of the document was in a different orientation - [issue #1312](https://github.com/py-pdf/fpdf2/issues/1213) ### Changed * improved logic for handling text substitution of the total number of pages, ensuring compatibility with text shaping - [issue #1090](https://github.com/py-pdf/fpdf2/issues/1090) * all [`AnnotationDict`](https://py-pdf.github.io/fpdf2/fpdf/annotations.html) properties can now be passed to `FPDF.text_annotation()`, `FPDF.free_text_annotation()`, `FPDF.add_action()`, `FPDF.add_text_markup_annotation()` & `FPDF.ink_annotation()`. This includes `title`, `color`, `border_width`... diff --git a/README.md b/README.md index cfb17dba5..96a0eb280 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![build status](https://github.com/py-pdf/fpdf2/workflows/build/badge.svg)](https://github.com/py-pdf/fpdf2/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/py-pdf/fpdf2/branch/master/graph/badge.svg)](https://codecov.io/gh/py-pdf/fpdf2) [![Pypi Trusted Publisher: enabled](https://img.shields.io/badge/Pypi%20Trusted%20Publisher-enabled-green.svg)](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) -[![checks: bandit, pylint, semgrep](https://img.shields.io/badge/checks-bandit,pylint,semgrep,grype-green.svg)](https://github.com/py-pdf/fpdf2/actions/workflows/continuous-integration-workflow.yml) +[![checks: bandit, guarddog, pylint, semgrep](https://img.shields.io/badge/checks-bandit,pylint,semgrep,grype-green.svg)](https://github.com/py-pdf/fpdf2/actions/workflows/continuous-integration-workflow.yml) [![Dependents](https://img.shields.io/librariesio/dependents/pypi/fpdf2)](https://libraries.io/pypi/fpdf2/dependents) [![Downloads per month](https://pepy.tech/badge/fpdf2/month)](https://pepy.tech/project/fpdf2) diff --git a/docs/Development.md b/docs/Development.md index e29c97b95..258421808 100644 --- a/docs/Development.md +++ b/docs/Development.md @@ -217,12 +217,13 @@ Ask maintainers through comments if some errors in the pipeline seem obscure to ### Release checklist 1. complete `CHANGELOG.md` and add the version & date of the new release 2. bump `FPDF_VERSION` in `fpdf/fpdf.py`. -Also (optionnal, once every year), update `contributors/contributors-map-small.png` based on +Also (optional, once every year), update `contributors/contributors-map-small.png` based on 3. update the `announce` block in `docs/overrides/main.html` to mention the new release 4. `git commit` & `git push` (if editing in a fork: submit and merge a PR) 5. check that [the GitHub Actions succeed](https://github.com/py-pdf/fpdf2/actions), and that [a new release appears on Pypi](https://pypi.org/project/fpdf2/#history) 6. perform a [GitHub release](https://github.com/py-pdf/fpdf2/releases), taking the description from the `CHANGELOG.md`. It will create a new `git` tag. +7. (optional) add a comment mentioning that the feature/fix has been released in all the GitHub issues mentioned in the `CHANGELOG.md` ## Documentation The standalone documentation is in the `docs` subfolder, diff --git a/docs/requirements.txt b/docs/requirements.txt index 467b777d0..5ed62fdeb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +lxml mkdocs mkdocs-include-markdown-plugin mkdocs-material diff --git a/test/image/image_types/test_insert_images.py b/test/image/image_types/test_insert_images.py index b9134d875..02f1b13e3 100644 --- a/test/image/image_types/test_insert_images.py +++ b/test/image/image_types/test_insert_images.py @@ -50,6 +50,10 @@ def test_insert_jpg_flatedecode(tmp_path): assert_pdf_equal(pdf, HERE / "image_types_insert_jpg_flatedecode.pdf", tmp_path) +@pytest.mark.skipif( + sys.version_info.minor != 13, + reason="LZWDecode is currently VERY slow, so we want to only execute it ONCE among all Python versions in the test pipeline", +) def test_insert_jpg_lzwdecode(tmp_path): pdf = fpdf.FPDF() pdf.compress = False From 4a577df017e1ec500227491cc241c8ab21024051 Mon Sep 17 00:00:00 2001 From: Lucas Cimon <925560+Lucas-C@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:55:54 +0100 Subject: [PATCH 2/2] Minor CI fixup --- .github/workflows/continuous-integration-workflow.yml | 5 +++++ README.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 7a602c798..4eb737b08 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -119,11 +119,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner + # Security hardening because this is a sensitive job, + # where extra care should be taken NOT to leak any secret uses: step-security/harden-runner@v2 with: egress-policy: block allowed-endpoints: github.com:443 + api.github.com:443 *.githubusercontent.com:443 pypi.org:443 files.pythonhosted.org:443 @@ -172,6 +175,8 @@ jobs: id-token: write steps: - name: Harden Runner + # Security hardening because this is a sensitive job, + # where extra care should be taken NOT to leak any secret uses: step-security/harden-runner@v2 with: egress-policy: block diff --git a/README.md b/README.md index 96a0eb280..1aa5b4339 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![build status](https://github.com/py-pdf/fpdf2/workflows/build/badge.svg)](https://github.com/py-pdf/fpdf2/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/py-pdf/fpdf2/branch/master/graph/badge.svg)](https://codecov.io/gh/py-pdf/fpdf2) [![Pypi Trusted Publisher: enabled](https://img.shields.io/badge/Pypi%20Trusted%20Publisher-enabled-green.svg)](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) -[![checks: bandit, guarddog, pylint, semgrep](https://img.shields.io/badge/checks-bandit,pylint,semgrep,grype-green.svg)](https://github.com/py-pdf/fpdf2/actions/workflows/continuous-integration-workflow.yml) +[![checks: bandit, grype, guarddog, pylint, semgrep](https://img.shields.io/badge/checks-bandit,grype,guarddog,pylint,semgrep-green.svg)](https://github.com/py-pdf/fpdf2/actions/workflows/continuous-integration-workflow.yml) [![Dependents](https://img.shields.io/librariesio/dependents/pypi/fpdf2)](https://libraries.io/pypi/fpdf2/dependents) [![Downloads per month](https://pepy.tech/badge/fpdf2/month)](https://pepy.tech/project/fpdf2)