From 0a7c09066254158e9caeebf84c1aca5eb70471d9 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 1 Aug 2024 13:56:03 +0200 Subject: [PATCH 01/37] Dockerfile: default for OCRD_MODULES was outdated --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4b6e91..0d9b579 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ ENV TF_FORCE_GPU_ALLOW_GROWTH=true # allow passing build-time parameter for list of tools to be installed # (defaults to medium, which also requires those modules to be present) -ARG OCRD_MODULES="core dinglehopper format-converters ocrd_calamari ocrd_cis ocrd_im6convert ocrd_keraslm ocrd_olena ocrd_segment ocrd_tesserocr tesseract tesserocr cor-asv-ann workflow-configuration" +ARG OCRD_MODULES="core cor-asv-ann dinglehopper docstruct format-converters nmalign ocrd_calamari ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_keraslm ocrd_olahd_client ocrd_olena ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_segment ocrd_tesserocr ocrd_wrap workflow-configuration" # persist that variable for build-time make to pick it up and run-time users to know what to expect ENV OCRD_MODULES="${OCRD_MODULES}" From 52a1b582879fbcd1a484f1b84677b2453195af35 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 10 Oct 2024 17:22:11 +0200 Subject: [PATCH 02/37] update 2024-10-11 --- CHANGELOG.md | 325 +++++++++++++++++++++++++++++++++++++++++ core | 2 +- dinglehopper | 2 +- docstruct | 2 +- eynollah | 2 +- nmalign | 2 +- ocrd_calamari | 2 +- ocrd_cis | 2 +- ocrd_detectron2 | 2 +- ocrd_doxa | 2 +- ocrd_fileformat | 2 +- ocrd_im6convert | 2 +- ocrd_keraslm | 2 +- ocrd_olahd_client | 2 +- ocrd_segment | 2 +- ocrd_tesserocr | 2 +- ocrd_wrap | 2 +- opencv-python | 2 +- sbb_binarization | 2 +- workflow-configuration | 2 +- 20 files changed, 344 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1095763..14af34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,331 @@ ## Unreleased +## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) + +### [core](https://github.com/OCR-D/core) [92b217e](https://github.com/OCR-D/core/commits/92b217e)..[85bde15](https://github.com/OCR-D/core/commits/85bde15) + +> Release: [v2.70.0](https://github.com/OCR-D/core/releases/v2.70.0) + + > * PyPI: do not upload deprecated distribution aliases anymore + > * deps-cuda: retry micro.mamba.pm even more + > * :package: v2.70.0 + > * :memo: changelog + > * create PyPI CD + > * :memo: changelog + > * Merge remote-tracking branch 'github/cli-decorator-import-network' + > * deps-cuda: retry if micromamba is unresponsive + > * Merge branch 'master' of https://github.com/OCR-D/core + > * :memo: changelog + > * Merge remote-tracking branch 'github/fix_mets_server_zombies' + > * :memo: changelog + > * Merge remote-tracking branch 'github/deps-torch-torchvision' + > * :memo: changelog + > * Merge branch 'network_client_block_prints' + > * Merge pull request #1280 from OCR-D/fix-docker-cuda-torch + > * :package: v2.69.0 + > * :memo: changelog + > * Merge branch 'mexthecat-master' + > * :memo: update changelog again + > * :memo: changelog: remove spurious entries + > * :memo: changelog + > * disableLogging: clearer comment + > * ocrd.cli.workspace: use physical_pages if possible, fix default output_field + > * OcrdMets.get_physical_pages: cover return_divs w/o for_fileIds for_pageIds + > * update OcrdPage from generateds + > * OcrdPage: add PageType.get_ReadingOrderGroups() + > * ocrd.cli.workspace: assert non-server in cmds mutating METS + > * tests: make sure ocrd_utils.config gets reset whenever changing it globally + > * lib.bash: fix errexit + > * run_processor: be robust if ocrd_tool is missing steps + > * ocrd.cli.validate tasks: pass on --mets-server-url, too + > * ocrd.cli.workspace server: add 'reload' and 'save' + > * ocrd.cli.workspace: consistently pass on --mets-server-url and --backup (also, simplify) + > * METS Server: also export+delegate physical_pages + > * PcGts.Page.id / make_xml_id: replace '/' with '_' + > * test_mets_server: add test for force (overwrite) + > * OcrdMetsServer.add_file: pass on 'force' kwarg, too + > * Workspace.reload_mets: fix for METS server case + > * add test for OcrdEnvConfig.reset_defaults() + > * ocrd_utils.config: add reset_defaults() + > * bashlib: re-add --log-filename, implement as stderr redirect + > * test-logging: also remove ocrd.log from tempdir + > * disableLogging: re-instate root logger, to + > * test_mets_server: use tmpdir to avoid side effects between suites + > * ClientSideOcrdMets: use same logger name prefix as server + > * pylint: try ignoring generateds (again) + > * update pylintrc + > * OcrdMets.add_agent: does not have positional args + > * cli.workspace: pass fileGrp as well, improve description + > * adapt to PIL.Image moved constants + > * fix exception + > * fix --log-filename (6fc606027a): apply in ocrd_cli_wrap_processor + > * tests report.is_valid: improve output on failure + > * Processor.zip_input_files: more verbose log msg + > * Processor.zip_input_files: warning instead of exception for missing input files + > * fix imports + > * ocrd_utils: forgot to export scale_coordinates at toplvl + > * allow "from ocrd_models import OcrdPage + > * improve output in case of assertion failures + > * hide/test expected deprecation warnings + > * use up-to-date kwargs (avoiding old deprecations) + > * mets_server: ClientSideOcrdMets needs OcrdMets-like kwargs (without deprecation) + > * test_mets_server: fix arg vs kwarg + > * processor CLI: delegate --resolve-resource, too + > * :package: v2.68.0 + > * :memo: changelog + > * refactor client cli: process -> run + > * Merge branch 'master' into extend-network-client + > * :memo: changelog + > * Merge pull request #1270 from OCR-D/fix-parsing + > * fix: exception handling + > * add: check processing job log file + > * add: discovery cli, processors and processor + > * add sort to network agents + > * add: parameter_override + > * fix: the annoying string dict + > * fix: check report validation outside try block + > * fix: set ps address if None in constructor + > * Fix: server_utils.py > 404 to 400 + > * Fix: rename to block + > * add docstring to cli commands + > * fix: required job id + > * add: help section to the cli + > * add cli job status check + > * add help for new env + > * refine status check methods + > * Update src/ocrd_network/client_utils.py + > * add timeout and wait to configs + > * add: client workflow run + > * fix: client processing request + > * fix test + > * refactor status checks + > * remove the client server + > * try docker host ip + > * Fix flag typo + > * integration test for client + > * update network client + > * fix the test dir path in docker + > * add integration test for client + > * Merge branch 'resolve-1257' + > * :memo: changelog + > * revert, and just use < v43.0.0 + > * set paramiko logging to INFO + > * fix: supress paramiko warnings + > * set: propagate 0, logging config + > * set: paramiko logging to ERROR + > * remove downloading tool json + > * add: default ocrd-all-tool.json + > * download tool json if missing + > * Merge branch 'master' into resolve-1257 + > * load tool json locally + +### [dinglehopper](https://github.com/qurator-spk/dinglehopper) [129e6eb](https://github.com/qurator-spk/dinglehopper/commits/129e6eb)..[2e6fe0c](https://github.com/qurator-spk/dinglehopper/commits/2e6fe0c) + +> Release: [v0.9.7](https://github.com/qurator-spk/dinglehopper/releases/v0.9.7) + + > * Merge pull request #113 from qurator-spk/python-3.13 + > * โœ” pre-commit: Add license check + > * ๐Ÿ› Fix --version option in OCR-D CLI + > * โœจ Support --version option in CLI + > * โš™ pyproject.toml: Add license.file + > * โš™ pre-commit: Update hooks + +### [docstruct](https://github.com/bertsky/docstruct) [a7ffdda](https://github.com/bertsky/docstruct/commits/a7ffdda)..[004e6ec](https://github.com/bertsky/docstruct/commits/004e6ec) + + > * add GHA CD via Dockerhub + +Submodule eynollah 032a99e...51f6ef6: + > * Merge pull request #137 from qurator-spk/dockerfile + > * Merge pull request #132 from qurator-spk/extracting_images_only + > * Merge pull request #133 from qurator-spk/src-layout + > * :package: v0.3.1 + > * :memo: changelog + > * Merge pull request #129 from qurator-spk/resolving_issue_106 + > * update Makefile model location + > * update pyproject.toml for v0.3.1 + > * update pyproject.toml + > * Update README.md + > * rename GH action + > * create draft pyproject.toml + > * format options table + > * Update README.md + > * improve huggingface url + > * remove CircleCI + > * Update model download url + > * Merge pull request #127 from bertsky/new-namespace-pkg + > * update GitHub actions + > * Update README.md + > * update supported Python+Tensorflow version combinations + > * pin tf2 version to 2.12.1 + > * use tf1 compatibility for keras backend + < adapt to OcrdFile.local_filename now :Path + < adapt to ocrd>=2.54 url vs local_filename + > * comment unnecessary print commands + > * add supported OS to readme + > * filtering separators in a correct way without missing them + > * Merge pull request #117 from qurator-spk/tf-2.12-or-greater + > * apply missed commit #a56988a back + > * Merge pull request #116 from qurator-spk/fix-typos + > * Merge pull request #113 from qurator-spk/tf_<2.12.0 + > * Update citation + > * Update bibtex entry + > * format citation info as bibtex + > * add HIP'23 paper reference + > * Merge pull request #109 from bertsky/patch-3 + > * Merge pull request #105 from bertsky/fix-model-archive-path + < Revert "Merge pull request #97 from qurator-spk/420-namespace-package" + > * Merge pull request #104 from bertsky/reinstate-namespace-pkg + > * Merge pull request #102 from qurator-spk/right2left_reading_order + > * delete printing resized image shape + > * issue #67 solved + > * improve links to GT guidelines + > * Update README.md + > * Update CHANGELOG.md + > * Update ocrd-tool.json + > * Merge pull request #86 from qurator-spk/eynollah_light + +### [nmalign](https://github.com/bertsky/nmalign) [7832c90](https://github.com/bertsky/nmalign/commits/7832c90)..[1426dbc](https://github.com/bertsky/nmalign/commits/1426dbc) + +> Release: [v0.0.3](https://github.com/bertsky/nmalign/releases/v0.0.3) + + > * fix dockerfile + > * add GHA CD via Dockerhub + +### [ocrd_calamari](https://github.com/OCR-D/ocrd_calamari) [caac953](https://github.com/OCR-D/ocrd_calamari/commits/caac953)..[d9cde1f](https://github.com/OCR-D/ocrd_calamari/commits/d9cde1f) + +> Release: [v1.0.6](https://github.com/OCR-D/ocrd_calamari/releases/v1.0.6) + + > * Merge pull request #116 from bertsky/limit-batch-size + > * Merge pull request #115 from OCR-D/pyproject_toml + > * โœ” CircleCI: Don't test on Python 3.7 anymore + > * โš™ pre-commit: Update hooks + > * ๐Ÿงน ruff: Update config + > * ๐ŸŽจ Reformat (Black) + > * ๐Ÿงน Update OCR-D API usage w.r.t. workspace.add_file() + > * โš™ pre-commit: Update hooks + +### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[fe122ae](https://github.com/cisocrgroup/ocrd_cis/commits/fe122ae) + +> Release: [v0.1.5](https://github.com/cisocrgroup/ocrd_cis/releases/v0.1.5) + + > * segment: adapt to numpy deprecation + > * Updated config.yml + > * CI: try testing in parallel + > * test_lib.bash: update GT Github URL + > * fix+update dockerfile + > * recognize: replace python-levenshtein with rapidfuzz + > * fix 53ae7d69 (already str not bytes) + > * use importlib instead of pkg_resources via ocrd_utils + > * docker: adapt to core using /build already + > * ocrd-tool.json: add Ocropy default model resources + > * resegment: fix 2 edge cases + +### [ocrd_detectron2](https://github.com/bertsky/ocrd_detectron2) [1f56273](https://github.com/bertsky/ocrd_detectron2/commits/1f56273)..[218e0b5](https://github.com/bertsky/ocrd_detectron2/commits/218e0b5) + +> Release: [v0.1.8](https://github.com/bertsky/ocrd_detectron2/releases/v0.1.8) + + > * CI: out py3.7, in py3.10 + > * CD: publish to ocrd/detectron2 instead of bertsky/ocrd_detectron2 + > * Docker: rebase to ocrd/core-cuda-torch + > * update from pillow deprecations + > * make deps: defer to detectron2 master (and always build from src) + +### [ocrd_doxa](https://github.com/bertsky/ocrd_doxa) [a95f8e7](https://github.com/bertsky/ocrd_doxa/commits/a95f8e7)..[15e8423](https://github.com/bertsky/ocrd_doxa/commits/15e8423) + +> Release: [v0.0.2](https://github.com/bertsky/ocrd_doxa/releases/v0.0.2) + + > * add DH badge + > * fix GHA username + > * fix GHA syntax + > * add Dockerhub CD + > * Merge pull request #5 from joschrew/dockerfile + +### [ocrd_fileformat](https://github.com/OCR-D/ocrd_fileformat) [ff46bd7](https://github.com/OCR-D/ocrd_fileformat/commits/ff46bd7)..[60cb696](https://github.com/OCR-D/ocrd_fileformat/commits/60cb696) + +> Release: [v0.11.1](https://github.com/OCR-D/ocrd_fileformat/releases/v0.11.1) + + > * test_convert: fix choice of assets (w/ wf. METS) + > * Merge pull request #60 from bertsky/patch-4 + > * Merge pull request #57 from OCR-D/license + +### [ocrd_im6convert](https://github.com/OCR-D/ocrd_im6convert) [be794f6](https://github.com/OCR-D/ocrd_im6convert/commits/be794f6)..[82bd491](https://github.com/OCR-D/ocrd_im6convert/commits/82bd491) + +> Release: [v0.1.1](https://github.com/OCR-D/ocrd_im6convert/releases/v0.1.1) + + > * CD: fix GHCR image tag + > * dockerfile: oops, confused im6convert with fileformat + > * Merge pull request #14 from OCR-D/docker-build-args + +### [ocrd_keraslm](https://github.com/OCR-D/ocrd_keraslm) [ea79b2a](https://github.com/OCR-D/ocrd_keraslm/commits/ea79b2a)..[2c466bd](https://github.com/OCR-D/ocrd_keraslm/commits/2c466bd) + +> Release: [v0.4.3](https://github.com/OCR-D/ocrd_keraslm/releases/v0.4.3) + + > * update assets + > * Revert "test: replace buggy workspace clone with cp -r" + > * test: replace buggy workspace clone with cp -r + > * docker: base on ocrd/core-cuda:2.69 (via env var) + +### [ocrd_olahd_client](https://github.com/OCR-D/ocrd_olahd_client) [6bcbb4b](https://github.com/OCR-D/ocrd_olahd_client/commits/6bcbb4b)..[56c9272](https://github.com/OCR-D/ocrd_olahd_client/commits/56c9272) + +> Release: [v0.0.2](https://github.com/OCR-D/ocrd_olahd_client/releases/v0.0.2) + + > * Merge branch 'dockerfile' + +### [ocrd_segment](https://github.com/OCR-D/ocrd_segment) [de824e9](https://github.com/OCR-D/ocrd_segment/commits/de824e9)..[064b7a8](https://github.com/OCR-D/ocrd_segment/commits/064b7a8) + +> Release: [v0.1.24](https://github.com/OCR-D/ocrd_segment/releases/v0.1.24) + + > * Merge pull request #68 from OCR-D/update-docker-2.67 + +### [ocrd_tesserocr](https://github.com/OCR-D/ocrd_tesserocr) [d23992b](https://github.com/OCR-D/ocrd_tesserocr/commits/d23992b)..[dcbd522](https://github.com/OCR-D/ocrd_tesserocr/commits/dcbd522) + +> Release: [v0.19.1](https://github.com/OCR-D/ocrd_tesserocr/releases/v0.19.1) + + > * Merge pull request #219 from OCR-D/fix-docker + > * GHA CD: forgot ghcr.io prefix + > * GHA CD: lowercase repo name + > * Merge pull request #215 from bertsky/circleci-cd-add-pypi + > * dockerfile: build Tesseract in parallel + > * dockerfile: adapt to weird dockerfile COPY syntax + > * docker: unblock+copy .gitmodules + > * dockerignore: unblock .git + > * dockerfile: copy .git repo as well (so submodule update works) during build + > * makefile: ensure submodules are checked out + > * makefile: fix clean-tesseract + > * Merge pull request #218 from OCR-D/update-docker-2.67 + +### [ocrd_wrap](https://github.com/bertsky/ocrd_wrap) [2cd800d](https://github.com/bertsky/ocrd_wrap/commits/2cd800d)..[fd4a2bc](https://github.com/bertsky/ocrd_wrap/commits/fd4a2bc) + +> Release: [v0.1.8](https://github.com/bertsky/ocrd_wrap/releases/v0.1.8) + + > * forgot Dockerfile + > * add GHA CD via Dockerhub + +### [opencv-python](https://github.com/skvark/opencv-python) [cce7c99](https://github.com/skvark/opencv-python/commits/cce7c99)..[6a181ce](https://github.com/skvark/opencv-python/commits/6a181ce) + +> Release: [84](https://github.com/skvark/opencv-python/releases/84) + + > * Merge pull request #1015 from Gornoka:patch-1 + +### [sbb_binarization](https://github.com/qurator-spk/sbb_binarization) [978f425](https://github.com/qurator-spk/sbb_binarization/commits/978f425)..[ccfc821](https://github.com/qurator-spk/sbb_binarization/commits/ccfc821) + +> Release: [v0.1.0](https://github.com/qurator-spk/sbb_binarization/releases/v0.1.0) + + > * remove shebang from setup.py (somehow breaking py39) + > * CI: increase memory on VM + > * make install: update setuptools, too + > * add GHA CD via Dockerhub + +### [workflow-configuration](https://github.com/bertsky/workflow-configuration) [eeea260](https://github.com/bertsky/workflow-configuration/commits/eeea260)..[63e9969](https://github.com/bertsky/workflow-configuration/commits/63e9969) + +> Release: [0.1.3](https://github.com/bertsky/workflow-configuration/releases/0.1.3) + + > * Adapt dockerimage to ocrd-d-core changes + > * :memo: update readme + > * new pair of XSLTs: un/flatten text regions in arbitrary regions + > * use mkdir -p when creating SHAREDIR + ## [v2024-07-17](https://github.com/OCR-D/ocrd_all/releases/v2024-07-17) Changes: diff --git a/core b/core index 92b217e..85bde15 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 92b217e36b68891724efe030542378ca995e4178 +Subproject commit 85bde1574293ea8b7ba29255fbb8e07312c28eb1 diff --git a/dinglehopper b/dinglehopper index 129e6eb..2e6fe0c 160000 --- a/dinglehopper +++ b/dinglehopper @@ -1 +1 @@ -Subproject commit 129e6eb427b0d5d306f76c7f443ee7cd08e83495 +Subproject commit 2e6fe0c2797081d3d91b406a10966c7ec6e86411 diff --git a/docstruct b/docstruct index a7ffdda..004e6ec 160000 --- a/docstruct +++ b/docstruct @@ -1 +1 @@ -Subproject commit a7ffdda68a4c9c4e0b0494e7b0f865d92297ac30 +Subproject commit 004e6ecf67860ae06c174bb1b349843632147aa2 diff --git a/eynollah b/eynollah index 032a99e..51f6ef6 160000 --- a/eynollah +++ b/eynollah @@ -1 +1 @@ -Subproject commit 032a99ef11b19d8cb97566a2ce086878544991a0 +Subproject commit 51f6ef63f5734ccb62447ad4c74f1336c0371bac diff --git a/nmalign b/nmalign index 7832c90..1426dbc 160000 --- a/nmalign +++ b/nmalign @@ -1 +1 @@ -Subproject commit 7832c9032a8ee5bee8104b80a8fa423e331ae728 +Subproject commit 1426dbc00c100118228bf6718fc746022845e029 diff --git a/ocrd_calamari b/ocrd_calamari index caac953..d9cde1f 160000 --- a/ocrd_calamari +++ b/ocrd_calamari @@ -1 +1 @@ -Subproject commit caac9532213f4685f5c4b738965c7b43fef04ccf +Subproject commit d9cde1f5c8b2957dae4fddb0df9b7188ebff1037 diff --git a/ocrd_cis b/ocrd_cis index 38ce45b..fe122ae 160000 --- a/ocrd_cis +++ b/ocrd_cis @@ -1 +1 @@ -Subproject commit 38ce45bf016546b748cce65031cad3fe24a35c0d +Subproject commit fe122ae4ac21e87e684af8c6b9aa02026bf0748c diff --git a/ocrd_detectron2 b/ocrd_detectron2 index 1f56273..218e0b5 160000 --- a/ocrd_detectron2 +++ b/ocrd_detectron2 @@ -1 +1 @@ -Subproject commit 1f56273d08fe098ac8b3f606c0a19927f8425225 +Subproject commit 218e0b5db7bf50880a22e1b0835f107a1c26c182 diff --git a/ocrd_doxa b/ocrd_doxa index a95f8e7..15e8423 160000 --- a/ocrd_doxa +++ b/ocrd_doxa @@ -1 +1 @@ -Subproject commit a95f8e77886c9860101392d088742ca0af277945 +Subproject commit 15e84239eb67ea89e827de3ec20ab64646b577c3 diff --git a/ocrd_fileformat b/ocrd_fileformat index ff46bd7..60cb696 160000 --- a/ocrd_fileformat +++ b/ocrd_fileformat @@ -1 +1 @@ -Subproject commit ff46bd7862096904c97cd612301a299d727ffd3f +Subproject commit 60cb696d6e4f662cb9e5bd4421cedac152729680 diff --git a/ocrd_im6convert b/ocrd_im6convert index be794f6..82bd491 160000 --- a/ocrd_im6convert +++ b/ocrd_im6convert @@ -1 +1 @@ -Subproject commit be794f67a2117ac3f8bcf42ce9eb496afbe33573 +Subproject commit 82bd49189e7c3c512059e4222ad7062e5a34f075 diff --git a/ocrd_keraslm b/ocrd_keraslm index ea79b2a..2c466bd 160000 --- a/ocrd_keraslm +++ b/ocrd_keraslm @@ -1 +1 @@ -Subproject commit ea79b2ab495c1d7ab3db678be27c89965b76e3b0 +Subproject commit 2c466bd6c07df5e6883672fdcbe5f48c0c3881e2 diff --git a/ocrd_olahd_client b/ocrd_olahd_client index 6bcbb4b..56c9272 160000 --- a/ocrd_olahd_client +++ b/ocrd_olahd_client @@ -1 +1 @@ -Subproject commit 6bcbb4bbb6847e581bdb84aa1c2c32b632d083f2 +Subproject commit 56c927282b3217e0eae2a41695974a9e94f43be8 diff --git a/ocrd_segment b/ocrd_segment index de824e9..064b7a8 160000 --- a/ocrd_segment +++ b/ocrd_segment @@ -1 +1 @@ -Subproject commit de824e9d5bb9a56ac253b7c6dd7d7c012cdddc64 +Subproject commit 064b7a86aff30677655c00af31c43b5b68c95fcd diff --git a/ocrd_tesserocr b/ocrd_tesserocr index d23992b..dcbd522 160000 --- a/ocrd_tesserocr +++ b/ocrd_tesserocr @@ -1 +1 @@ -Subproject commit d23992b0068e6fa9612cddcfe106628461538442 +Subproject commit dcbd5227f834c219e36d5dc41606a08e85e67a15 diff --git a/ocrd_wrap b/ocrd_wrap index 2cd800d..fd4a2bc 160000 --- a/ocrd_wrap +++ b/ocrd_wrap @@ -1 +1 @@ -Subproject commit 2cd800d9eccbc084751558a87972ac22ee60e87a +Subproject commit fd4a2bcab2dfd18f831b0e101763b3438afe1c8b diff --git a/opencv-python b/opencv-python index cce7c99..6a181ce 160000 --- a/opencv-python +++ b/opencv-python @@ -1 +1 @@ -Subproject commit cce7c994d46406205eb39300bb7ca9c48d80185a +Subproject commit 6a181cedae5f23e332ae87d2e90d36635932855e diff --git a/sbb_binarization b/sbb_binarization index 978f425..ccfc821 160000 --- a/sbb_binarization +++ b/sbb_binarization @@ -1 +1 @@ -Subproject commit 978f425bd154458e92888d2a974fe759bb3a5c06 +Subproject commit ccfc8212ec7a0238ef4eeda638ebf78bb03f758c diff --git a/workflow-configuration b/workflow-configuration index eeea260..63e9969 160000 --- a/workflow-configuration +++ b/workflow-configuration @@ -1 +1 @@ -Subproject commit eeea2609a8d5415537a5e26d4d662e1f24d7f3c2 +Subproject commit 63e996995238a4e1f60204b0039ba4dfcdc56880 From 016ab068a1a8e8c56ed391ae97cc44816bac52da Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 10 Oct 2024 18:21:51 +0200 Subject: [PATCH 03/37] actually add ocrd_page2alto --- .gitmodules | 3 +++ Makefile | 9 ++++++++- ocrd_page2alto | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 160000 ocrd_page2alto diff --git a/.gitmodules b/.gitmodules index e63bf02..62d1779 100644 --- a/.gitmodules +++ b/.gitmodules @@ -94,3 +94,6 @@ [submodule "ocrd_froc"] path = ocrd_froc url = https://github.com/OCR-D/ocrd_froc +[submodule "ocrd_page2alto"] + path = ocrd_page2alto + url = https://github.com/OCR-D/page-to-alto diff --git a/Makefile b/Makefile index bd109da..d7458dd 100644 --- a/Makefile +++ b/Makefile @@ -298,6 +298,13 @@ $(call multirule,$(OCRD_DETECTRON2)): ocrd_detectron2 $(BIN)/ocrd | $(OCRD_KRAKE $(pip_install) endif +ifneq ($(filter ocrd_page2alto, $(OCRD_MODULES)),) +OCRD_EXECUTABLES += $(OCRD_PAGE_TO_ALTO) +OCRD_PAGE_TO_ALTO := $(BIN)/ocrd-page2alto-transform +$(OCRD_PAGE_TO_ALTO): ocrd_page2alto $(BIN)/ocrd + $(pip_install) +endif + ifneq ($(filter ocrd_ocropy, $(OCRD_MODULES)),) OCRD_EXECUTABLES += $(OCRD_OCROPY) OCRD_OCROPY := $(BIN)/ocrd-ocropy-segment @@ -847,7 +854,7 @@ dockers: docker-minimum docker-minimum-cuda docker-medium docker-medium-cuda doc docker-%: PIP_OPTIONS = -e # Minimum-size selection: use Ocropy binarization, use Tesseract from git -docker-mini%: DOCKER_MODULES := core ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_tesserocr ocrd_wrap workflow-configuration ocrd_olahd_client +docker-mini%: DOCKER_MODULES := core ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_tesserocr ocrd_wrap workflow-configuration ocrd_olahd_client ocrd_page2alto # Medium-size selection: add Olena binarization and Calamari, add evaluation docker-medi%: DOCKER_MODULES := core cor-asv-ann dinglehopper docstruct format-converters nmalign ocrd_calamari ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_keraslm ocrd_olahd_client ocrd_olena ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_segment ocrd_tesserocr ocrd_wrap workflow-configuration # Maximum-size selection: use all modules diff --git a/ocrd_page2alto b/ocrd_page2alto new file mode 160000 index 0000000..8877e8f --- /dev/null +++ b/ocrd_page2alto @@ -0,0 +1 @@ +Subproject commit 8877e8f050df76448ff99bf27f43994b30f894ba From 76494e21ddc4bb8cdb04e0d6244ab769087d2904 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 11 Oct 2024 16:55:19 +0200 Subject: [PATCH 04/37] update ocrd_cis --- CHANGELOG.md | 8 ++++++++ ocrd_cis | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14af34e..353fc8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) + +### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[db65d7f](https://github.com/cisocrgroup/ocrd_cis/commits/db65d7f) + +> Release: [v0.1.5](https://github.com/cisocrgroup/ocrd_cis/releases/v0.1.5) + + ## Unreleased ## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) @@ -210,6 +217,7 @@ Submodule eynollah 032a99e...51f6ef6: > Release: [v0.1.5](https://github.com/cisocrgroup/ocrd_cis/releases/v0.1.5) + > * Merge pull request #6 from MehmedGIT/fix-align-import-levenshtein > * segment: adapt to numpy deprecation > * Updated config.yml > * CI: try testing in parallel diff --git a/ocrd_cis b/ocrd_cis index fe122ae..db65d7f 160000 --- a/ocrd_cis +++ b/ocrd_cis @@ -1 +1 @@ -Subproject commit fe122ae4ac21e87e684af8c6b9aa02026bf0748c +Subproject commit db65d7fbea89d9fe8055eddb302eeb6c7294159e From 0d2cd1ad343a3f14e95757101b17988a8b89f75a Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 11 Oct 2024 17:00:44 +0200 Subject: [PATCH 05/37] update ocrd_fileformat --- CHANGELOG.md | 8 ++++++++ ocrd_fileformat | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353fc8f..0ca1328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) + +### [ocrd_fileformat](https://github.com/OCR-D/ocrd_fileformat) [ff46bd7](https://github.com/OCR-D/ocrd_fileformat/commits/ff46bd7)..[8ab078d](https://github.com/OCR-D/ocrd_fileformat/commits/8ab078d) + +> Release: [v0.11.1](https://github.com/OCR-D/ocrd_fileformat/releases/v0.11.1) + + ## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) ### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[db65d7f](https://github.com/cisocrgroup/ocrd_cis/commits/db65d7f) @@ -254,6 +261,7 @@ Submodule eynollah 032a99e...51f6ef6: > Release: [v0.11.1](https://github.com/OCR-D/ocrd_fileformat/releases/v0.11.1) + > * update ocr-fileformat to include UB-Mannheim/ocr-fileformat#188 > * test_convert: fix choice of assets (w/ wf. METS) > * Merge pull request #60 from bertsky/patch-4 > * Merge pull request #57 from OCR-D/license diff --git a/ocrd_fileformat b/ocrd_fileformat index 60cb696..8ab078d 160000 --- a/ocrd_fileformat +++ b/ocrd_fileformat @@ -1 +1 @@ -Subproject commit 60cb696d6e4f662cb9e5bd4421cedac152729680 +Subproject commit 8ab078dee246b0715db40a15ef7d288668febdf2 From adbad4237653e2ed67072d7aae8560501fcdafda Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:21:20 +0200 Subject: [PATCH 06/37] try to fix conflicts in main venv - ocrd-calamari: no need to reinstall protobuf here anymore - fix-cuda: downgrade Torch (to avoid conflicts around typing-extensions) --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d7458dd..f8a5f42 100644 --- a/Makefile +++ b/Makefile @@ -546,8 +546,6 @@ install-models-calamari: $(BIN)/ocrd OCRD_EXECUTABLES += $(OCRD_CALAMARI) OCRD_CALAMARI := $(BIN)/ocrd-calamari-recognize $(OCRD_CALAMARI): ocrd_calamari $(BIN)/ocrd - @# workaround for Calamari#337: - . $(ACTIVATE_VENV) && $(SEMPIP) pip install "protobuf<4" $(pip_install) endif @@ -822,7 +820,8 @@ deps-cuda: core $(ACTIVATE_VENV) tf1nvidia: $(ACTIVATE_VENV) $(pip_install_tf1nvidia) -# post-fix workaround for clash between cuDNN of Tensorflow 2.12 (โ†’8.6) and Pytorch 1.13 (โ†’8.5) +# post-fix workaround for clash between cuDNN of Tensorflow 2.12 (โ†’8.6) and Pytorch 1.13 (โ†’8.5) / 2.1 (8.7) +# (which also involves conflict around typing-extensions version) # the latter is explicit (but unnecessary), the former is implicit (and causes "DNN library not found" crashes at runtime) # so we have three potential options: # 1. revert to the version required by TF after pip overruled our choice via Torch dependency @@ -831,9 +830,9 @@ tf1nvidia: $(ACTIVATE_VENV) # pip3 install "tensorflow<2.12" # 3. upgrade Torch so there is no overt conflict # pip install "torch>=2.0" -# Since ATM we don't know whether Torch 2.x will work everywhere, we opt for 2: +# Since ATM we already need TF 2.12, we choose for (modified) option 3: fix-cuda: $(ACTIVATE_VENV) - . $(ACTIVATE_VENV) && $(SEMPIP) pip install "tensorflow<2.12" + . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 .PHONY: deps-cuda tf1nvidia fix-cuda From 2ec1937ce8b2963abdf9babdd51a289322e41d83 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:03:14 +0200 Subject: [PATCH 07/37] more attempts - ensure semaphore when creating venvs, too - fix-cuda: same in sub-venv (avoid CUDA 12) --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f8a5f42..d48e62f 100644 --- a/Makefile +++ b/Makefile @@ -210,8 +210,8 @@ $(BIN)/pip: $(ACTIVATE_VENV) . $(ACTIVATE_VENV) && $(SEMPIP) pip install --upgrade pip setuptools %/bin/activate: - $(PYTHON) -m venv $(subst /bin/activate,,$@) - . $@ && pip install --upgrade pip setuptools wheel + $(SEMPIP) $(PYTHON) -m venv $(subst /bin/activate,,$@) + . $@ && $(SEMPIP) pip install --upgrade pip setuptools wheel .PHONY: wheel wheel: $(BIN)/wheel @@ -833,6 +833,9 @@ tf1nvidia: $(ACTIVATE_VENV) # Since ATM we already need TF 2.12, we choose for (modified) option 3: fix-cuda: $(ACTIVATE_VENV) . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 +# displace CUDA 12 libs pulled via Pytorch from PyPI + . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch~=2.1.0+cu118 + .PHONY: deps-cuda tf1nvidia fix-cuda From e1ee446422e5138660075c0e85c82d496a7b7412 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:18:05 +0200 Subject: [PATCH 08/37] fix-cuda: not in sub-venv if it does not exist, yet --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d48e62f..b36ea6d 100644 --- a/Makefile +++ b/Makefile @@ -834,8 +834,9 @@ tf1nvidia: $(ACTIVATE_VENV) fix-cuda: $(ACTIVATE_VENV) . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 # displace CUDA 12 libs pulled via Pytorch from PyPI - . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch~=2.1.0+cu118 - + if test -d $(SUB_VENV_TF1); then \ + . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch~=2.1.0+cu118 \ + pip check; fi .PHONY: deps-cuda tf1nvidia fix-cuda From f093e5da94468fcda7a7e5402a0b8a7c5a2ca9fb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:41:45 +0200 Subject: [PATCH 09/37] fix-cuda: local version label needs exact requirement --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b36ea6d..3e488ce 100644 --- a/Makefile +++ b/Makefile @@ -835,7 +835,7 @@ fix-cuda: $(ACTIVATE_VENV) . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 # displace CUDA 12 libs pulled via Pytorch from PyPI if test -d $(SUB_VENV_TF1); then \ - . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch~=2.1.0+cu118 \ + . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 \ pip check; fi .PHONY: deps-cuda tf1nvidia fix-cuda From 6855e1f497d4429676acfb4e7f50cfe8e2c87c7c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 01:15:20 +0200 Subject: [PATCH 10/37] fix-cuda: fix typo in recipe --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3e488ce..63e295d 100644 --- a/Makefile +++ b/Makefile @@ -835,8 +835,8 @@ fix-cuda: $(ACTIVATE_VENV) . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 # displace CUDA 12 libs pulled via Pytorch from PyPI if test -d $(SUB_VENV_TF1); then \ - . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118 \ - pip check; fi + . $(SUB_VENV_TF1)/bin/activate && pip install -i https://download.pytorch.org/whl/cu118 torchvision==0.16.2+cu118 torch==2.1.2+cu118; \ + fi .PHONY: deps-cuda tf1nvidia fix-cuda From 8eac021623babfb22ff98689899a54d160c4e305 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 02:22:47 +0200 Subject: [PATCH 11/37] docker: fix mini/medi defs --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 63e295d..341b8b9 100644 --- a/Makefile +++ b/Makefile @@ -288,12 +288,9 @@ endif endif ifneq ($(filter ocrd_detectron2, $(OCRD_MODULES)),) -# ocrd_detectron patches detectron2 until there is a new detectron2 release. -# See https://github.com/facebookresearch/detectron2/pull/5011 for details. -CUSTOM_DEPS += patch OCRD_EXECUTABLES += $(OCRD_DETECTRON2) OCRD_DETECTRON2 := $(BIN)/ocrd-detectron2-segment -$(call multirule,$(OCRD_DETECTRON2)): ocrd_detectron2 $(BIN)/ocrd | $(OCRD_KRAKEN) +$(OCRD_DETECTRON2): ocrd_detectron2 $(BIN)/ocrd | $(OCRD_KRAKEN) . $(ACTIVATE_VENV) && $(MAKE) -C $< deps $(pip_install) endif @@ -301,7 +298,8 @@ endif ifneq ($(filter ocrd_page2alto, $(OCRD_MODULES)),) OCRD_EXECUTABLES += $(OCRD_PAGE_TO_ALTO) OCRD_PAGE_TO_ALTO := $(BIN)/ocrd-page2alto-transform -$(OCRD_PAGE_TO_ALTO): ocrd_page2alto $(BIN)/ocrd +OCRD_PAGE_TO_ALTO += $(BIN)/page-to-alto +$(call multirule,$(OCRD_PAGE_TO_ALTO)): ocrd_page2alto $(BIN)/ocrd $(pip_install) endif @@ -857,10 +855,12 @@ dockers: docker-minimum docker-minimum-cuda docker-medium docker-medium-cuda doc docker-%: PIP_OPTIONS = -e # Minimum-size selection: use Ocropy binarization, use Tesseract from git -docker-mini%: DOCKER_MODULES := core ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_tesserocr ocrd_wrap workflow-configuration ocrd_olahd_client ocrd_page2alto +DOCKER_MODULES_MINI := core ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_olahd_client ocrd_page2alto ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_tesserocr ocrd_wrap workflow-configuration +docker-mini%: DOCKER_MODULES := $(DOCKER_MODULES_MINI) # Medium-size selection: add Olena binarization and Calamari, add evaluation -docker-medi%: DOCKER_MODULES := core cor-asv-ann dinglehopper docstruct format-converters nmalign ocrd_calamari ocrd_cis ocrd_fileformat ocrd_im6convert ocrd_keraslm ocrd_olahd_client ocrd_olena ocrd_pagetopdf ocrd_repair_inconsistencies ocrd_segment ocrd_tesserocr ocrd_wrap workflow-configuration -# Maximum-size selection: use all modules +DOCKER_MODULES_MEDI := $(DOCKER_MODULES_MINI) cor-asv-ann dinglehopper docstruct format-converters nmalign ocrd_calamari ocrd_keraslm ocrd_olena ocrd_segment +docker-medi%: DOCKER_MODULES := $(DOCKER_MODULES_MEDI) +# Maximum-size selection: use all enabled modules docker-maxi%: DOCKER_MODULES := $(OCRD_MODULES) # DOCKER_BASE_IMAGE From 1bed4e369c4f691c567a9e31932b78b36caa3be6 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 03:10:15 +0200 Subject: [PATCH 12/37] CI/CD: try to split up mini/medi/maxi into inter-dependent jobs --- .circleci/config.yml | 212 +++++++++++++++++++++++++++++++++---------- 1 file changed, 165 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e282a6..050b6df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,39 +1,28 @@ version: 2.1 jobs: - build: + build-mini: docker: - image: cimg/base:current-22.04 - resource_class: large + parameters: + variant: + type: + string steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker: + docker_layer_caching: true - run: - name: build image - command: make docker-maximum-cuda GIT_DEPTH=--single-branch DOCKER_PARALLEL=-j4 + name: Build Docker image + command: make docker-minim<< parameters.variant >> GIT_DEPTH=--single-branch no_output_timeout: 30m - run: name: test image command: | mkdir test-results - docker run --rm -v $PWD:/data ocrd/all:maximum-cuda make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + docker run --rm -v $PWD:/data ocrd/all:minim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: path: test-results - - when: - # takes too long for 1h1m CircleCI timeout overall - # also, storage is limited... - condition: false - steps: - - run: - name: persist image - command: | - sudo apt install pigz - docker image save ocrd/all:maximum-cuda | pigz --fast > ocrd-all-maximum.tar.gz - no_output_timeout: 30m - # can be downloaded from CircleCI.com and imported via "docker image load" - - store_artifacts: - path: ocrd-all-maximum.tar.gz - destination: artifacts - deploy: + build-medi: docker: - image: cimg/base:current-22.04 parameters: @@ -42,15 +31,100 @@ jobs: string steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build Docker image + # override dependency (should come from cached layer) in make: + command: make docker-medi<< parameters.variant >> -o docker-minim<< parameters.variant >> GIT_DEPTH=--single-branch + no_output_timeout: 30m + - run: + name: test image + command: | + mkdir test-results + docker run --rm -v $PWD:/data ocrd/all:medi<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + - store_test_results: + path: test-results + build-maxi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true - run: name: Build Docker image - command: make docker-<< parameters.variant >> GIT_DEPTH=--single-branch - # fails due to pip races: DOCKER_PARALLEL=-j3 + # override dependency (should come from cached layer) in make: + command: make docker-maxim<< parameters.variant >> -o docker-medi<< parameters.variant >> GIT_DEPTH=--single-branch no_output_timeout: 30m - run: - name: Alias Docker images - command: docker tag ocrd/all:<< parameters.variant >> ocrd/all:<< parameters.variant >>-git + name: test image + command: | + mkdir test-results + docker run --rm -v $PWD:/data ocrd/all:maxim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + - store_test_results: + path: test-results + deploy-mini: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - checkout + - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - run: + name: Alias Docker image + command: docker tag ocrd/all:minim<< parameters.variant >> ocrd/all:minim<< parameters.variant >>-git + - run: + name: Login to Docker Hub + command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + - run: + name: Push images to Docker Hub + no_output_timeout: 2.5h + command: | + docker push ocrd/all:minim<< parameters.variant >> + docker push ocrd/all:minim<< parameters.variant >>-git + deploy-medi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - checkout + - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - run: + name: Alias Docker image + command: docker tag ocrd/all:medi<< parameters.variant >> ocrd/all:medi<< parameters.variant >>-git + - run: + name: Login to Docker Hub + command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + - run: + name: Push images to Docker Hub + no_output_timeout: 2.5h + command: | + docker push ocrd/all:medi<< parameters.variant >> + docker push ocrd/all:medi<< parameters.variant >>-git + deploy-maxi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - checkout + - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - run: + name: Alias Docker image + command: docker tag ocrd/all:maxim<< parameters.variant >> ocrd/all:maxim<< parameters.variant >>-git - run: name: Login to Docker Hub command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin @@ -58,21 +132,12 @@ jobs: name: Push images to Docker Hub no_output_timeout: 2.5h command: | - docker push ocrd/all:<< parameters.variant >> - docker push ocrd/all:<< parameters.variant >>-git + docker push ocrd/all:maxim<< parameters.variant >> + docker push ocrd/all:maxim<< parameters.variant >>-git - when: condition: - equal: [ maximum, << parameters.variant >> ] + equal: [ um, << parameters.variant >> ] steps: - - run: - name: Alias and push intermediate variants - command: | - docker tag ocrd/all:medium ocrd/all:medium-git - docker tag ocrd/all:minimum ocrd/all:minimum-git - docker push ocrd/all:minimum - docker push ocrd/all:minimum-git - docker push ocrd/all:medium - docker push ocrd/all:medium-git - run: name: store ocrd-all-tool.json command: | @@ -91,7 +156,7 @@ jobs: destination: ocrd-all-module-dir.json - when: condition: - equal: [ maximum-cuda, << parameters.variant >> ] + equal: [ um-cuda, << parameters.variant >> ] steps: - run: name: Create a date-versioned mirror of ocrd/all:maximum-cuda @@ -99,20 +164,73 @@ jobs: - run: name: Update badge command: curl -X POST "$MICROBADGER_WEBHOOK" || true + - when: + # takes too long for 1h1m CircleCI timeout overall + # also, storage is limited... + condition: false + steps: + - run: + name: persist image + command: | + sudo apt install pigz + docker image save ocrd/all:maxim<< parameters.variant >> | pigz --fast > ocrd-all-maximum.tar.gz + no_output_timeout: 30m + # can be downloaded from CircleCI.com and imported via "docker image load" + - store_artifacts: + path: ocrd-all-maximum.tar.gz + destination: artifacts + workflows: version: 2 - build-master: + build: jobs: - - deploy: + - build-mini: + name: build-minim<< matrix.variant >> matrix: parameters: - variant: [maximum, maximum-cuda] + variant: [um, um-cuda] + - build-medi: + name: build-medi<< matrix.variant >> + matrix: + parameters: + variant: [um, um-cuda] + requires: + - build-minim<< matrix.variant >> + - build-maxi: + name: build-maxim<< matrix.variant >> + matrix: + parameters: + variant: [um, um-cuda] + requires: + - build-medi<< matrix.variant >> + - deploy-mini: + name: deploy-minim<< matrix.variant >> + matrix: + parameters: + variant: [um, um-cuda] + requires: + - build-minim<< matrix.variant >> filters: branches: only: master - build-pull-request: - jobs: - - build: + - deploy-medi: + name: deploy-medi<< matrix.variant >> + matrix: + parameters: + variant: [um, um-cuda] + requires: + - build-medi<< matrix.variant >> filters: branches: - ignore: master + only: master + - deploy-maxi: + name: deploy-maxim<< matrix.variant >> + matrix: + parameters: + variant: [um, um-cuda] + requires: + - build-maxim<< matrix.variant >> + filters: + branches: + only: master + From 9ee4e818d2248c4d6aea35045723d1032f8748cf Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 04:02:12 +0200 Subject: [PATCH 13/37] CI/CD: store_test_results with file path --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 050b6df..4ff08d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: mkdir test-results docker run --rm -v $PWD:/data ocrd/all:minim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: - path: test-results + path: test-results/core.xml build-medi: docker: - image: cimg/base:current-22.04 @@ -44,7 +44,7 @@ jobs: mkdir test-results docker run --rm -v $PWD:/data ocrd/all:medi<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: - path: test-results + path: test-results/core.xml build-maxi: docker: - image: cimg/base:current-22.04 @@ -67,7 +67,7 @@ jobs: mkdir test-results docker run --rm -v $PWD:/data ocrd/all:maxim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: - path: test-results + path: test-results/core.xml deploy-mini: docker: - image: cimg/base:current-22.04 From c49e6c6ad38790fe4a87eb09d4d16d18dbb3cca9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 12 Oct 2024 04:02:13 +0200 Subject: [PATCH 14/37] docker: do not rm venv created by previous stage --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0d9b579..73df1c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,9 +83,6 @@ RUN echo "Acquire::http::Timeout \"3000\";" >> /etc/apt/apt.conf.d/99network && WORKDIR /build -# create virtual environment -RUN rm $VIRTUAL_ENV/bin/pip* && apt-get purge -y python3-pip && python3 -m venv $VIRTUAL_ENV && python3 -m pip install --force pip - # from-stage already contains a clone clashing with build context RUN rm -rf /build/core/.git From fab9caa0f66fc9f56bab8295b89de0f70f67a4d0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 01:41:21 +0200 Subject: [PATCH 15/37] Updated config.yml --- .circleci/config.yml | 61 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ff08d4..f2ce265 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,7 @@ jobs: string steps: - checkout - - setup_remote_docker: - docker_layer_caching: true + - setup_remote_docker - run: name: Build Docker image command: make docker-minim<< parameters.variant >> GIT_DEPTH=--single-branch @@ -22,6 +21,15 @@ jobs: docker run --rm -v $PWD:/data ocrd/all:minim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: path: test-results/core.xml + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/minim<< parameters.variant >>.tar + - persist_to_workspace: + root: . + paths: + - images build-medi: docker: - image: cimg/base:current-22.04 @@ -31,8 +39,12 @@ jobs: string steps: - checkout - - setup_remote_docker: - docker_layer_caching: true + - setup_remote_docker + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/minim<< parameters.variant >>.tar - run: name: Build Docker image # override dependency (should come from cached layer) in make: @@ -45,6 +57,15 @@ jobs: docker run --rm -v $PWD:/data ocrd/all:medi<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: path: test-results/core.xml + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/medi<< parameters.variant >>.tar + - persist_to_workspace: + root: . + paths: + - images build-maxi: docker: - image: cimg/base:current-22.04 @@ -54,8 +75,12 @@ jobs: string steps: - checkout - - setup_remote_docker: - docker_layer_caching: true + - setup_remote_docker + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/medi<< parameters.variant >>.tar - run: name: Build Docker image # override dependency (should come from cached layer) in make: @@ -68,6 +93,15 @@ jobs: docker run --rm -v $PWD:/data ocrd/all:maxim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: path: test-results/core.xml + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/maxim<< parameters.variant >>.tar + - persist_to_workspace: + root: . + paths: + - images deploy-mini: docker: - image: cimg/base:current-22.04 @@ -78,6 +112,11 @@ jobs: steps: - checkout - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/minim<< parameters.variant >>.tar - run: name: Alias Docker image command: docker tag ocrd/all:minim<< parameters.variant >> ocrd/all:minim<< parameters.variant >>-git @@ -100,6 +139,11 @@ jobs: steps: - checkout - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/medi<< parameters.variant >>.tar - run: name: Alias Docker image command: docker tag ocrd/all:medi<< parameters.variant >> ocrd/all:medi<< parameters.variant >>-git @@ -122,6 +166,11 @@ jobs: steps: - checkout - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/maxim<< parameters.variant >>.tar - run: name: Alias Docker image command: docker tag ocrd/all:maxim<< parameters.variant >> ocrd/all:maxim<< parameters.variant >>-git From 4ce8a0b35192ec20bea5ac64ff52acb1373aa4ff Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 02:02:52 +0200 Subject: [PATCH 16/37] CI/CD: transfer images via workspace and docker save/load --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2ce265..8b99a2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: name: save image command: | mkdir -p images - docker image save -o images/minim<< parameters.variant >>.tar + docker image save -o images/minim<< parameters.variant >>.tar ocrd/all:minim<< parameters.variant >> - persist_to_workspace: root: . paths: @@ -61,7 +61,7 @@ jobs: name: save image command: | mkdir -p images - docker image save -o images/medi<< parameters.variant >>.tar + docker image save -o images/medi<< parameters.variant >>.tar ocrd/all:medi<< parameters.variant >> - persist_to_workspace: root: . paths: @@ -97,7 +97,7 @@ jobs: name: save image command: | mkdir -p images - docker image save -o images/maxim<< parameters.variant >>.tar + docker image save -o images/maxim<< parameters.variant >>.tar ocrd/all:maxim<< parameters.variant >> - persist_to_workspace: root: . paths: From f5b7b6bd08eb3dd66ab7a485427d59f509d4cd80 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 03:38:08 +0200 Subject: [PATCH 17/37] CI/CD: allow long wait for saving/persisting Docker image --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b99a2d..b6925cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,10 +26,12 @@ jobs: command: | mkdir -p images docker image save -o images/minim<< parameters.variant >>.tar ocrd/all:minim<< parameters.variant >> + no_output_timeout: 30m - persist_to_workspace: root: . paths: - images + no_output_timeout: 30m build-medi: docker: - image: cimg/base:current-22.04 @@ -62,10 +64,12 @@ jobs: command: | mkdir -p images docker image save -o images/medi<< parameters.variant >>.tar ocrd/all:medi<< parameters.variant >> + no_output_timeout: 30m - persist_to_workspace: root: . paths: - images + no_output_timeout: 30m build-maxi: docker: - image: cimg/base:current-22.04 @@ -98,10 +102,12 @@ jobs: command: | mkdir -p images docker image save -o images/maxim<< parameters.variant >>.tar ocrd/all:maxim<< parameters.variant >> + no_output_timeout: 30m - persist_to_workspace: root: . paths: - images + no_output_timeout: 30m deploy-mini: docker: - image: cimg/base:current-22.04 From 5766a7383d6e3a3aa23d8521b7619ebbc378fe68 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:24:29 +0200 Subject: [PATCH 18/37] try to fix conflicts in main venv, and some - contenders: protobuf and typing-extensions - culprits: torch and tensorflow --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 341b8b9..ed8b3d6 100644 --- a/Makefile +++ b/Makefile @@ -292,6 +292,8 @@ OCRD_EXECUTABLES += $(OCRD_DETECTRON2) OCRD_DETECTRON2 := $(BIN)/ocrd-detectron2-segment $(OCRD_DETECTRON2): ocrd_detectron2 $(BIN)/ocrd | $(OCRD_KRAKEN) . $(ACTIVATE_VENV) && $(MAKE) -C $< deps + # pre-empt conflict around typing-extensions + . $(ACTIVATE_VENV) && $(SEMPIP) pip install -i https://download.pytorch.org/whl/cpu torchvision==0.16.2 torch==2.1.2 $(pip_install) endif @@ -600,6 +602,8 @@ SBB_BINARIZATION := $(BIN)/ocrd-sbb-binarize SBB_BINARIZATION += $(BIN)/sbb_binarize $(call multirule,$(SBB_BINARIZATION)): sbb_binarization $(BIN)/ocrd $(pip_install) + # work around #67 - switch to version pinned by eynollah: + . $(ACTIVATE_VENV) && $(SEMPIP) pip install "tensorflow==2.12.1" endif ifneq ($(filter eynollah, $(OCRD_MODULES)),) @@ -611,6 +615,8 @@ OCRD_EXECUTABLES += $(EYNOLLAH_SEGMENT) EYNOLLAH_SEGMENT := $(BIN)/ocrd-eynollah-segment $(EYNOLLAH_SEGMENT): eynollah $(BIN)/ocrd $(pip_install) + # solve conflict with ocrd_calamari: + . $(ACTIVATE_VENV) && $(SEMPIP) pip install "protobuf<4" endif ifneq ($(filter ocrd_repair_inconsistencies, $(OCRD_MODULES)),) From 08e67b81d61af2148f0a6717598e66e370ef6382 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:07:03 +0200 Subject: [PATCH 19/37] CI/CD: move image transfers to pre-steps and post-steps, use command defs --- .circleci/config.yml | 312 +++++++++++++++++++++---------------------- 1 file changed, 151 insertions(+), 161 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6925cf..f5d2505 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,131 +1,45 @@ version: 2.1 -jobs: - build-mini: - docker: - - image: cimg/base:current-22.04 + +commands: + build: + description: "template for all docker-* build jobs" parameters: - variant: + tag: + description: "image tag (full variant name) to be built" type: string - steps: - - checkout - - setup_remote_docker - - run: - name: Build Docker image - command: make docker-minim<< parameters.variant >> GIT_DEPTH=--single-branch - no_output_timeout: 30m - - run: - name: test image - command: | - mkdir test-results - docker run --rm -v $PWD:/data ocrd/all:minim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - - store_test_results: - path: test-results/core.xml - - run: - name: save image - command: | - mkdir -p images - docker image save -o images/minim<< parameters.variant >>.tar ocrd/all:minim<< parameters.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images - no_output_timeout: 30m - build-medi: - docker: - - image: cimg/base:current-22.04 - parameters: - variant: + deptag: + description: "image tag (full variant name) this depends on (should _not_ be rebuilt)" type: string + default: "core" steps: - checkout - setup_remote_docker - - attach_workspace: - at: . - - run: - name: load previous image - command: docker image load < images/minim<< parameters.variant >>.tar - run: name: Build Docker image - # override dependency (should come from cached layer) in make: - command: make docker-medi<< parameters.variant >> -o docker-minim<< parameters.variant >> GIT_DEPTH=--single-branch + command: make docker-<< parameters.tag >> -o docker-<< parameters.deptag >> GIT_DEPTH=--single-branch no_output_timeout: 30m - run: name: test image command: | mkdir test-results - docker run --rm -v $PWD:/data ocrd/all:medi<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - store_test_results: path: test-results/core.xml - - run: - name: save image - command: | - mkdir -p images - docker image save -o images/medi<< parameters.variant >>.tar ocrd/all:medi<< parameters.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images - no_output_timeout: 30m - build-maxi: - docker: - - image: cimg/base:current-22.04 + deploy: + description: "template for all docker-* deploy jobs" parameters: - variant: + tag: + description: "image tag (full variant name) to be pushed" type: string steps: - checkout - setup_remote_docker - - attach_workspace: - at: . - - run: - name: load previous image - command: docker image load < images/medi<< parameters.variant >>.tar - - run: - name: Build Docker image - # override dependency (should come from cached layer) in make: - command: make docker-maxim<< parameters.variant >> -o docker-medi<< parameters.variant >> GIT_DEPTH=--single-branch - no_output_timeout: 30m - - run: - name: test image - command: | - mkdir test-results - docker run --rm -v $PWD:/data ocrd/all:maxim<< parameters.variant >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - - store_test_results: - path: test-results/core.xml - - run: - name: save image - command: | - mkdir -p images - docker image save -o images/maxim<< parameters.variant >>.tar ocrd/all:maxim<< parameters.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images - no_output_timeout: 30m - deploy-mini: - docker: - - image: cimg/base:current-22.04 - parameters: - variant: - type: - string - steps: - - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ - - attach_workspace: - at: . - - run: - name: load previous image - command: docker image load < images/minim<< parameters.variant >>.tar - run: name: Alias Docker image - command: docker tag ocrd/all:minim<< parameters.variant >> ocrd/all:minim<< parameters.variant >>-git + command: docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-git - run: name: Login to Docker Hub command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin @@ -133,65 +47,11 @@ jobs: name: Push images to Docker Hub no_output_timeout: 2.5h command: | - docker push ocrd/all:minim<< parameters.variant >> - docker push ocrd/all:minim<< parameters.variant >>-git - deploy-medi: - docker: - - image: cimg/base:current-22.04 - parameters: - variant: - type: - string - steps: - - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ - - attach_workspace: - at: . - - run: - name: load previous image - command: docker image load < images/medi<< parameters.variant >>.tar - - run: - name: Alias Docker image - command: docker tag ocrd/all:medi<< parameters.variant >> ocrd/all:medi<< parameters.variant >>-git - - run: - name: Login to Docker Hub - command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - - run: - name: Push images to Docker Hub - no_output_timeout: 2.5h - command: | - docker push ocrd/all:medi<< parameters.variant >> - docker push ocrd/all:medi<< parameters.variant >>-git - deploy-maxi: - docker: - - image: cimg/base:current-22.04 - parameters: - variant: - type: - string - steps: - - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ - - attach_workspace: - at: . - - run: - name: load previous image - command: docker image load < images/maxim<< parameters.variant >>.tar - - run: - name: Alias Docker image - command: docker tag ocrd/all:maxim<< parameters.variant >> ocrd/all:maxim<< parameters.variant >>-git - - run: - name: Login to Docker Hub - command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - - run: - name: Push images to Docker Hub - no_output_timeout: 2.5h - command: | - docker push ocrd/all:maxim<< parameters.variant >> - docker push ocrd/all:maxim<< parameters.variant >>-git + docker push ocrd/all:<< parameters.tag >> + docker push ocrd/all:<< parameters.tag >>-git - when: condition: - equal: [ um, << parameters.variant >> ] + equal: [ maximum, << parameters.tag >> ] steps: - run: name: store ocrd-all-tool.json @@ -211,7 +71,7 @@ jobs: destination: ocrd-all-module-dir.json - when: condition: - equal: [ um-cuda, << parameters.variant >> ] + equal: [ maximum-cuda, << parameters.tag >> ] steps: - run: name: Create a date-versioned mirror of ocrd/all:maximum-cuda @@ -228,12 +88,76 @@ jobs: name: persist image command: | sudo apt install pigz - docker image save ocrd/all:maxim<< parameters.variant >> | pigz --fast > ocrd-all-maximum.tar.gz + docker image save ocrd/all:<< parameters.tag >> | pigz --fast > ocrd-all-maximum.tar.gz no_output_timeout: 30m # can be downloaded from CircleCI.com and imported via "docker image load" - store_artifacts: path: ocrd-all-maximum.tar.gz destination: artifacts + +jobs: + build-mini: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + tag: minim<< parameters.variant >> + build-medi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + deptag: minim<< parameters.variant >> + tag: medi<< parameters.variant >> + build-maxi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + deptag: medi<< parameters.variant >> + tag: maxim<< parameters.variant >> + deploy-mini: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + tag: minim<< parameters.variant >> + deploy-medi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + tag: medi<< parameters.variant >> + deploy-maxi: + docker: + - image: cimg/base:current-22.04 + parameters: + variant: + type: + string + steps: + - build: + tag: maxim<< parameters.variant >> workflows: version: 2 @@ -244,6 +168,18 @@ workflows: matrix: parameters: variant: [um, um-cuda] + post-steps: + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/minim<< matrix.variant >>.tar ocrd/all:minim<< matrix.variant >> + no_output_timeout: 30m + - persist_to_workspace: + root: . + paths: + - images + no_output_timeout: 30m - build-medi: name: build-medi<< matrix.variant >> matrix: @@ -251,6 +187,24 @@ workflows: variant: [um, um-cuda] requires: - build-minim<< matrix.variant >> + pre-steps: + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/minim<< matrix.variant >>.tar + post-steps: + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/medi<< matrix.variant >>.tar ocrd/all:medi<< matrix.variant >> + no_output_timeout: 30m + - persist_to_workspace: + root: . + paths: + - images + no_output_timeout: 30m - build-maxi: name: build-maxim<< matrix.variant >> matrix: @@ -258,6 +212,24 @@ workflows: variant: [um, um-cuda] requires: - build-medi<< matrix.variant >> + pre-steps: + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/medi<< matrix.variant >>.tar + post-steps: + - run: + name: save image + command: | + mkdir -p images + docker image save -o images/maxim<< matrix.variant >>.tar ocrd/all:maxim<< matrix.variant >> + no_output_timeout: 30m + - persist_to_workspace: + root: . + paths: + - images + no_output_timeout: 30m - deploy-mini: name: deploy-minim<< matrix.variant >> matrix: @@ -265,6 +237,12 @@ workflows: variant: [um, um-cuda] requires: - build-minim<< matrix.variant >> + pre-steps: + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/minim<< matrix.variant >>.tar filters: branches: only: master @@ -275,6 +253,12 @@ workflows: variant: [um, um-cuda] requires: - build-medi<< matrix.variant >> + pre-steps: + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/medi<< matrix.variant >>.tar filters: branches: only: master @@ -285,6 +269,12 @@ workflows: variant: [um, um-cuda] requires: - build-maxim<< matrix.variant >> + pre-steps: + - attach_workspace: + at: . + - run: + name: load previous image + command: docker image load < images/maxim<< matrix.variant >>.tar filters: branches: only: master From a94646dca2369a12bc50c8188b940dad71e199c2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:39:02 +0200 Subject: [PATCH 20/37] CI/CD: rm images after loading to Docker daemon (to avoid conflict on checkout) --- .circleci/config.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5d2505..6386f01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -192,7 +192,9 @@ workflows: at: . - run: name: load previous image - command: docker image load < images/minim<< matrix.variant >>.tar + command: | + docker image load < images/minim<< matrix.variant >>.tar + rm -fr images post-steps: - run: name: save image @@ -217,7 +219,9 @@ workflows: at: . - run: name: load previous image - command: docker image load < images/medi<< matrix.variant >>.tar + command: | + docker image load < images/medi<< matrix.variant >>.tar + rm -fr images post-steps: - run: name: save image @@ -242,7 +246,9 @@ workflows: at: . - run: name: load previous image - command: docker image load < images/minim<< matrix.variant >>.tar + command: | + docker image load < images/minim<< matrix.variant >>.tar + rm -fr images filters: branches: only: master @@ -258,7 +264,9 @@ workflows: at: . - run: name: load previous image - command: docker image load < images/medi<< matrix.variant >>.tar + command: | + docker image load < images/medi<< matrix.variant >>.tar + rm -fr images filters: branches: only: master @@ -274,7 +282,9 @@ workflows: at: . - run: name: load previous image - command: docker image load < images/maxim<< matrix.variant >>.tar + command: | + docker image load < images/maxim<< matrix.variant >>.tar + rm -fr images filters: branches: only: master From 5746a546115c438f7ea583a3d9850912e62a4b7b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 14 Oct 2024 14:51:51 +0000 Subject: [PATCH 21/37] update sbb_binarization to #70 --- sbb_binarization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbb_binarization b/sbb_binarization index ccfc821..4eabd12 160000 --- a/sbb_binarization +++ b/sbb_binarization @@ -1 +1 @@ -Subproject commit ccfc8212ec7a0238ef4eeda638ebf78bb03f758c +Subproject commit 4eabd125a3366982d930eba0d728615f49af9cee From 70394050773e1611cafb22e029160d86c3e7125b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 14 Oct 2024 14:52:21 +0000 Subject: [PATCH 22/37] update dinglehopper to #120 --- dinglehopper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dinglehopper b/dinglehopper index 2e6fe0c..071e6a8 160000 --- a/dinglehopper +++ b/dinglehopper @@ -1 +1 @@ -Subproject commit 2e6fe0c2797081d3d91b406a10966c7ec6e86411 +Subproject commit 071e6a8bd14b47dd65c21a6acaa278677b0f352b From 21b2e342846d2f401c89e073810d5367afe821a6 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 14 Oct 2024 18:03:31 +0200 Subject: [PATCH 23/37] CI/CD: abuse Dockerhub for transferring Docker images between jobs This is my first attempt at utilising the v2 API (for removal of temporary tags on Dockerhub). - already contained: remove temporary images after deploy - still missing: cleanup after build: - in case of failure: immediately - normally: if not on master, then when - on medium: remove minimum - on maximum: remove medium - _after_ maximum: remove maximum (the problem with this is that the `docker push` is implemented as a `post-step` in the workflow def) --- .circleci/config.yml | 78 +++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6386f01..dd9e9d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,18 @@ version: 2.1 commands: + dockerhub-remove: + description: "remove an image on Dockerhub" + parameters: + tag: + description: "image tag (full variant name) to be removed" + type: + string + steps: + - run: + command: | + HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>/ build: description: "template for all docker-* build jobs" parameters: @@ -172,13 +184,8 @@ workflows: - run: name: save image command: | - mkdir -p images - docker image save -o images/minim<< matrix.variant >>.tar ocrd/all:minim<< matrix.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images + docker tag ocrd/all:minim<< matrix.variant >> ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 + docker push ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 no_output_timeout: 30m - build-medi: name: build-medi<< matrix.variant >> @@ -188,24 +195,17 @@ workflows: requires: - build-minim<< matrix.variant >> pre-steps: - - attach_workspace: - at: . - run: name: load previous image command: | - docker image load < images/minim<< matrix.variant >>.tar - rm -fr images + docker pull ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 + docker tag ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:minim<< matrix.variant >> post-steps: - run: name: save image command: | - mkdir -p images - docker image save -o images/medi<< matrix.variant >>.tar ocrd/all:medi<< matrix.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images + docker tag ocrd/all:medi<< matrix.variant >> ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 + docker push ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 no_output_timeout: 30m - build-maxi: name: build-maxim<< matrix.variant >> @@ -215,24 +215,17 @@ workflows: requires: - build-medi<< matrix.variant >> pre-steps: - - attach_workspace: - at: . - run: name: load previous image command: | - docker image load < images/medi<< matrix.variant >>.tar - rm -fr images + docker pull ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 + docker tag ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:medi<< matrix.variant >> post-steps: - run: name: save image command: | - mkdir -p images - docker image save -o images/maxim<< matrix.variant >>.tar ocrd/all:maxim<< matrix.variant >> - no_output_timeout: 30m - - persist_to_workspace: - root: . - paths: - - images + docker tag ocrd/all:maxim<< matrix.variant >> ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 + docker push ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 no_output_timeout: 30m - deploy-mini: name: deploy-minim<< matrix.variant >> @@ -242,13 +235,14 @@ workflows: requires: - build-minim<< matrix.variant >> pre-steps: - - attach_workspace: - at: . - run: name: load previous image command: | - docker image load < images/minim<< matrix.variant >>.tar - rm -fr images + docker pull ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 + docker tag ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:minim<< matrix.variant >> + post-steps: + - dockerhub-remove: + tag: minim<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master @@ -260,13 +254,14 @@ workflows: requires: - build-medi<< matrix.variant >> pre-steps: - - attach_workspace: - at: . - run: name: load previous image command: | - docker image load < images/medi<< matrix.variant >>.tar - rm -fr images + docker pull ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 + docker tag ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:medi<< matrix.variant >> + post-steps: + - dockerhub-remove: + tag: medi<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master @@ -278,13 +273,14 @@ workflows: requires: - build-maxim<< matrix.variant >> pre-steps: - - attach_workspace: - at: . - run: name: load previous image command: | - docker image load < images/maxim<< matrix.variant >>.tar - rm -fr images + docker pull ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 + docker tag ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:maxim<< matrix.variant >> + post-steps: + - dockerhub-remove: + tag: maxim<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master From edea0c536d352a0c2b3d546a5556834a0cfb47ae Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 01:03:10 +0200 Subject: [PATCH 24/37] CI/CD: work around CircleCI not allowing variables in workflow ctxt --- .circleci/config.yml | 123 ++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 67 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd9e9d8..356bcf0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,33 @@ version: 2.1 commands: + dockerhub-store: + description: "store an image on Dockerhub" + parameters: + tag: + description: "image tag (full variant name) to be removed" + type: + string + steps: + - run: + name: save image + command: | + docker tag ocrd/all:<< matrix.tag >> ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} + docker push ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} + no_output_timeout: 30m + dockerhub-load: + description: "load an image from Dockerhub" + parameters: + tag: + description: "image tag (full variant name) to be pulled" + type: + string + steps: + - run: + name: load previous image + command: | + docker pull ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} + docker tag ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} ocrd/all:<< matrix.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: @@ -12,7 +39,7 @@ commands: - run: command: | HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) - curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>/ + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/ build: description: "template for all docker-* build jobs" parameters: @@ -20,7 +47,7 @@ commands: description: "image tag (full variant name) to be built" type: string - deptag: + dep: description: "image tag (full variant name) this depends on (should _not_ be rebuilt)" type: string @@ -118,6 +145,8 @@ jobs: steps: - build: tag: minim<< parameters.variant >> + - dockerhub-store: + tag: minim<< parameters.variant >> build-medi: docker: - image: cimg/base:current-22.04 @@ -126,8 +155,12 @@ jobs: type: string steps: + - dockerhub-load: + tag: minim<< parameters.variant >> - build: - deptag: minim<< parameters.variant >> + dep: minim<< parameters.variant >> + tag: medi<< parameters.variant >> + - dockerhub-store: tag: medi<< parameters.variant >> build-maxi: docker: @@ -137,8 +170,12 @@ jobs: type: string steps: + - dockerhub-load: + tag: medi<< parameters.variant >> - build: - deptag: medi<< parameters.variant >> + dep: medi<< parameters.variant >> + tag: maxim<< parameters.variant >> + - dockerhub-store: tag: maxim<< parameters.variant >> deploy-mini: docker: @@ -148,7 +185,11 @@ jobs: type: string steps: - - build: + - dockerhub-load: + tag: minim<< parameters.variant >> + - deploy: + tag: minim<< parameters.variant >> + - dockerhub-remove: tag: minim<< parameters.variant >> deploy-medi: docker: @@ -158,7 +199,11 @@ jobs: type: string steps: - - build: + - dockerhub-load: + tag: medi<< parameters.variant >> + - deploy: + tag: medi<< parameters.variant >> + - dockerhub-remove: tag: medi<< parameters.variant >> deploy-maxi: docker: @@ -168,7 +213,11 @@ jobs: type: string steps: - - build: + - dockerhub-load: + tag: maxim<< parameters.variant >> + - deploy: + tag: maxim<< parameters.variant >> + - dockerhub-remove: tag: maxim<< parameters.variant >> workflows: @@ -180,13 +229,6 @@ workflows: matrix: parameters: variant: [um, um-cuda] - post-steps: - - run: - name: save image - command: | - docker tag ocrd/all:minim<< matrix.variant >> ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 - docker push ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 - no_output_timeout: 30m - build-medi: name: build-medi<< matrix.variant >> matrix: @@ -194,19 +236,6 @@ workflows: variant: [um, um-cuda] requires: - build-minim<< matrix.variant >> - pre-steps: - - run: - name: load previous image - command: | - docker pull ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 - docker tag ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:minim<< matrix.variant >> - post-steps: - - run: - name: save image - command: | - docker tag ocrd/all:medi<< matrix.variant >> ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 - docker push ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 - no_output_timeout: 30m - build-maxi: name: build-maxim<< matrix.variant >> matrix: @@ -214,19 +243,6 @@ workflows: variant: [um, um-cuda] requires: - build-medi<< matrix.variant >> - pre-steps: - - run: - name: load previous image - command: | - docker pull ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 - docker tag ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:medi<< matrix.variant >> - post-steps: - - run: - name: save image - command: | - docker tag ocrd/all:maxim<< matrix.variant >> ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 - docker push ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 - no_output_timeout: 30m - deploy-mini: name: deploy-minim<< matrix.variant >> matrix: @@ -234,15 +250,6 @@ workflows: variant: [um, um-cuda] requires: - build-minim<< matrix.variant >> - pre-steps: - - run: - name: load previous image - command: | - docker pull ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 - docker tag ocrd/all:minim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:minim<< matrix.variant >> - post-steps: - - dockerhub-remove: - tag: minim<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master @@ -253,15 +260,6 @@ workflows: variant: [um, um-cuda] requires: - build-medi<< matrix.variant >> - pre-steps: - - run: - name: load previous image - command: | - docker pull ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 - docker tag ocrd/all:medi<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:medi<< matrix.variant >> - post-steps: - - dockerhub-remove: - tag: medi<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master @@ -272,15 +270,6 @@ workflows: variant: [um, um-cuda] requires: - build-maxim<< matrix.variant >> - pre-steps: - - run: - name: load previous image - command: | - docker pull ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 - docker tag ocrd/all:maxim<< matrix.variant >>-$CIRCLE_SHA1 ocrd/all:maxim<< matrix.variant >> - post-steps: - - dockerhub-remove: - tag: maxim<< matrix.variant >>-$CIRCLE_SHA1 filters: branches: only: master From 2ae2842bb6eb61d6a299dac8a7786af7551aaa3e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 01:24:00 +0200 Subject: [PATCH 25/37] CI/CD: fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 356bcf0..8acec54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ commands: - setup_remote_docker - run: name: Build Docker image - command: make docker-<< parameters.tag >> -o docker-<< parameters.deptag >> GIT_DEPTH=--single-branch + command: make docker-<< parameters.tag >> -o docker-<< parameters.dep >> GIT_DEPTH=--single-branch no_output_timeout: 30m - run: name: test image From 03fe0f8d6f6aafb98ee8a62b2cf7a8c9737ce8e2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 01:25:30 +0200 Subject: [PATCH 26/37] CI/CD: fix another typo --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8acec54..055e5ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,8 +12,8 @@ commands: - run: name: save image command: | - docker tag ocrd/all:<< matrix.tag >> ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} - docker push ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} + docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} + docker push ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} no_output_timeout: 30m dockerhub-load: description: "load an image from Dockerhub" @@ -26,8 +26,8 @@ commands: - run: name: load previous image command: | - docker pull ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} - docker tag ocrd/all:<< matrix.tag >>-${CIRCLE_SHA1} ocrd/all:<< matrix.tag >> + docker pull ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} + docker tag ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} ocrd/all:<< parameters.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: From 9a5542af0e7cf41892d64dedd4dea82ceadab82b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 02:25:03 +0200 Subject: [PATCH 27/37] CI/CD: try with other form of parameter+variable interpolation --- .circleci/config.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 055e5ab..7cd0d82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,12 +8,14 @@ commands: description: "image tag (full variant name) to be removed" type: string + environment: + - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} steps: - run: name: save image command: | - docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} - docker push ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} + docker tag ocrd/all:<< parameters.tag >> ocrd/all:$SHA1TAG + docker push ocrd/all:$SHA1TAG no_output_timeout: 30m dockerhub-load: description: "load an image from Dockerhub" @@ -22,12 +24,14 @@ commands: description: "image tag (full variant name) to be pulled" type: string + environment: + - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} steps: - run: name: load previous image command: | - docker pull ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} - docker tag ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} ocrd/all:<< parameters.tag >> + docker pull ocrd/all:$SHA1TAG + docker tag ocrd/all:$SHA1TAG ocrd/all:<< parameters.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: @@ -35,11 +39,13 @@ commands: description: "image tag (full variant name) to be removed" type: string + environment: + - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} steps: - run: command: | HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) - curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/ + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/$SHA1TAG/ build: description: "template for all docker-* build jobs" parameters: From 49d57da1194d040a5b93ffde41268bbd770b47c9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:10:04 +0200 Subject: [PATCH 28/37] CI/CD: CircleCI notoriously does not interpolate variables in commands --- .circleci/config.yml | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cd0d82..593b533 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,17 +5,19 @@ commands: description: "store an image on Dockerhub" parameters: tag: - description: "image tag (full variant name) to be removed" + description: "image tag (full variant name) to be saved" + type: + string + sha: + description: "image tag (temporary alias) to save as" type: string - environment: - - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} steps: - run: name: save image command: | - docker tag ocrd/all:<< parameters.tag >> ocrd/all:$SHA1TAG - docker push ocrd/all:$SHA1TAG + docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.sha >> + docker push ocrd/all:<< parameters.sha >> no_output_timeout: 30m dockerhub-load: description: "load an image from Dockerhub" @@ -24,28 +26,28 @@ commands: description: "image tag (full variant name) to be pulled" type: string - environment: - - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} + sha: + description: "image tag (temporary alias) to pull as" + type: + string steps: - run: name: load previous image command: | - docker pull ocrd/all:$SHA1TAG - docker tag ocrd/all:$SHA1TAG ocrd/all:<< parameters.tag >> + docker pull ocrd/all:<< parameters.sha >> + docker tag ocrd/all:<< parameters.sha >> ocrd/all:<< parameters.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: - tag: - description: "image tag (full variant name) to be removed" + sha: + description: "image tag (temporary alias) to be removed" type: string - environment: - - SHA1TAG: << parameters.tag >>-${CIRCLE_SHA1} steps: - run: command: | HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) - curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/$SHA1TAG/ + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.sha >>/ build: description: "template for all docker-* build jobs" parameters: @@ -153,6 +155,7 @@ jobs: tag: minim<< parameters.variant >> - dockerhub-store: tag: minim<< parameters.variant >> + sha: minim<< parameters.variant >>-${CIRCLE_SHA1} build-medi: docker: - image: cimg/base:current-22.04 @@ -162,12 +165,14 @@ jobs: string steps: - dockerhub-load: + sha: minim<< parameters.variant >>-${CIRCLE_SHA1} tag: minim<< parameters.variant >> - build: dep: minim<< parameters.variant >> tag: medi<< parameters.variant >> - dockerhub-store: tag: medi<< parameters.variant >> + sha: medi<< parameters.variant >>-${CIRCLE_SHA1} build-maxi: docker: - image: cimg/base:current-22.04 @@ -177,12 +182,14 @@ jobs: string steps: - dockerhub-load: + sha: medi<< parameters.variant >>-${CIRCLE_SHA1} tag: medi<< parameters.variant >> - build: dep: medi<< parameters.variant >> tag: maxim<< parameters.variant >> - dockerhub-store: tag: maxim<< parameters.variant >> + sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} deploy-mini: docker: - image: cimg/base:current-22.04 @@ -192,11 +199,12 @@ jobs: string steps: - dockerhub-load: + sha: minim<< parameters.variant >>-${CIRCLE_SHA1} tag: minim<< parameters.variant >> - deploy: tag: minim<< parameters.variant >> - dockerhub-remove: - tag: minim<< parameters.variant >> + sha: minim<< parameters.variant >>-${CIRCLE_SHA1} deploy-medi: docker: - image: cimg/base:current-22.04 @@ -206,11 +214,12 @@ jobs: string steps: - dockerhub-load: + sha: medi<< parameters.variant >>-${CIRCLE_SHA1} tag: medi<< parameters.variant >> - deploy: tag: medi<< parameters.variant >> - dockerhub-remove: - tag: medi<< parameters.variant >> + sha: medi<< parameters.variant >>-${CIRCLE_SHA1} deploy-maxi: docker: - image: cimg/base:current-22.04 @@ -220,11 +229,12 @@ jobs: string steps: - dockerhub-load: + sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} tag: maxim<< parameters.variant >> - deploy: tag: maxim<< parameters.variant >> - dockerhub-remove: - tag: maxim<< parameters.variant >> + sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} workflows: version: 2 From 80bc708c280ee68eb11f3a98cabc47957545d247 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:39:44 +0200 Subject: [PATCH 29/37] CI/CD: try to adapt to CircleCI's stupid variable scoping --- .circleci/config.yml | 51 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 593b533..a6abd8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,16 +8,13 @@ commands: description: "image tag (full variant name) to be saved" type: string - sha: - description: "image tag (temporary alias) to save as" - type: - string steps: - run: name: save image command: | - docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.sha >> - docker push ocrd/all:<< parameters.sha >> + TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} + docker tag ocrd/all:<< parameters.tag >> ocrd/all:$TMPTAG + docker push ocrd/all:$TMPTAG no_output_timeout: 30m dockerhub-load: description: "load an image from Dockerhub" @@ -26,28 +23,26 @@ commands: description: "image tag (full variant name) to be pulled" type: string - sha: - description: "image tag (temporary alias) to pull as" - type: - string steps: - run: name: load previous image command: | - docker pull ocrd/all:<< parameters.sha >> - docker tag ocrd/all:<< parameters.sha >> ocrd/all:<< parameters.tag >> + TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} + docker pull ocrd/all:$TMPTAG + docker tag ocrd/all:$TMPTAG ocrd/all:<< parameters.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: - sha: - description: "image tag (temporary alias) to be removed" + tag: + description: "image tag (full variant name) to be removed" type: string steps: - run: command: | + TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) - curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.sha >>/ + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/$TMPTAG/ build: description: "template for all docker-* build jobs" parameters: @@ -151,11 +146,12 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - build: tag: minim<< parameters.variant >> - dockerhub-store: tag: minim<< parameters.variant >> - sha: minim<< parameters.variant >>-${CIRCLE_SHA1} build-medi: docker: - image: cimg/base:current-22.04 @@ -164,15 +160,15 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: - sha: minim<< parameters.variant >>-${CIRCLE_SHA1} tag: minim<< parameters.variant >> - build: dep: minim<< parameters.variant >> tag: medi<< parameters.variant >> - dockerhub-store: tag: medi<< parameters.variant >> - sha: medi<< parameters.variant >>-${CIRCLE_SHA1} build-maxi: docker: - image: cimg/base:current-22.04 @@ -181,15 +177,15 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: - sha: medi<< parameters.variant >>-${CIRCLE_SHA1} tag: medi<< parameters.variant >> - build: dep: medi<< parameters.variant >> tag: maxim<< parameters.variant >> - dockerhub-store: tag: maxim<< parameters.variant >> - sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} deploy-mini: docker: - image: cimg/base:current-22.04 @@ -198,13 +194,14 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: - sha: minim<< parameters.variant >>-${CIRCLE_SHA1} tag: minim<< parameters.variant >> - deploy: tag: minim<< parameters.variant >> - dockerhub-remove: - sha: minim<< parameters.variant >>-${CIRCLE_SHA1} + tag: minim<< parameters.variant >> deploy-medi: docker: - image: cimg/base:current-22.04 @@ -213,13 +210,14 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: - sha: medi<< parameters.variant >>-${CIRCLE_SHA1} tag: medi<< parameters.variant >> - deploy: tag: medi<< parameters.variant >> - dockerhub-remove: - sha: medi<< parameters.variant >>-${CIRCLE_SHA1} + tag: medi<< parameters.variant >> deploy-maxi: docker: - image: cimg/base:current-22.04 @@ -228,13 +226,14 @@ jobs: type: string steps: + - run: + command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: - sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} tag: maxim<< parameters.variant >> - deploy: tag: maxim<< parameters.variant >> - dockerhub-remove: - sha: maxim<< parameters.variant >>-${CIRCLE_SHA1} + tag: maxim<< parameters.variant >> workflows: version: 2 From 13f7da8e72aab88ed36cbd4220c108d81a646515 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:00:36 +0200 Subject: [PATCH 30/37] CI/CD: debug what is going on with envvars here? --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6abd8d..bcc0a70 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,7 @@ commands: - run: name: save image command: | + set -x TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} docker tag ocrd/all:<< parameters.tag >> ocrd/all:$TMPTAG docker push ocrd/all:$TMPTAG @@ -67,6 +68,7 @@ commands: command: | mkdir test-results docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + ls -ltr test-results - store_test_results: path: test-results/core.xml deploy: From 7b32661bd10940df19aed9a1282495fcc3e62ddb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 15 Oct 2024 11:49:35 +0000 Subject: [PATCH 31/37] update sbb_binarization to #70, again --- sbb_binarization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbb_binarization b/sbb_binarization index 4eabd12..d259795 160000 --- a/sbb_binarization +++ b/sbb_binarization @@ -1 +1 @@ -Subproject commit 4eabd125a3366982d930eba0d728615f49af9cee +Subproject commit d2597959bab3e1c6910775b506327be1c5a8f256 From 27f06a80323e225f9f676dfe7a43fcefc1e59bd5 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 15 Oct 2024 12:54:25 +0000 Subject: [PATCH 32/37] CI/CD: simply forgot to login to Dockerhub --- .circleci/config.yml | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bcc0a70..ab3f822 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,13 +9,14 @@ commands: type: string steps: + - run: + name: Login to Docker Hub + command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - run: name: save image command: | - set -x - TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} - docker tag ocrd/all:<< parameters.tag >> ocrd/all:$TMPTAG - docker push ocrd/all:$TMPTAG + docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} + docker push ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} no_output_timeout: 30m dockerhub-load: description: "load an image from Dockerhub" @@ -28,9 +29,8 @@ commands: - run: name: load previous image command: | - TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} - docker pull ocrd/all:$TMPTAG - docker tag ocrd/all:$TMPTAG ocrd/all:<< parameters.tag >> + docker pull ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} + docker tag ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} ocrd/all:<< parameters.tag >> dockerhub-remove: description: "remove an image on Dockerhub" parameters: @@ -41,9 +41,8 @@ commands: steps: - run: command: | - TMPTAG=<< parameters.tag >>-${CIRCLE_SHA1:0:8} HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) - curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/$TMPTAG/ + curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/ build: description: "template for all docker-* build jobs" parameters: @@ -68,7 +67,6 @@ commands: command: | mkdir test-results docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml - ls -ltr test-results - store_test_results: path: test-results/core.xml deploy: @@ -148,8 +146,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - build: tag: minim<< parameters.variant >> - dockerhub-store: @@ -162,8 +158,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: tag: minim<< parameters.variant >> - build: @@ -179,8 +173,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: tag: medi<< parameters.variant >> - build: @@ -196,8 +188,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: tag: minim<< parameters.variant >> - deploy: @@ -212,8 +202,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: tag: medi<< parameters.variant >> - deploy: @@ -228,8 +216,6 @@ jobs: type: string steps: - - run: - command: echo 'export CIRCLE_SHA1="$CIRCLE_SHA1"' >> $BASH_ENV - dockerhub-load: tag: maxim<< parameters.variant >> - deploy: From f41d394cc18fcabac535f4f3134b4bbac6a8a4d1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 15 Oct 2024 12:54:47 +0000 Subject: [PATCH 33/37] CI/CD: store core test results via docker-cp --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab3f822..e3346e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,7 +66,12 @@ commands: name: test image command: | mkdir test-results - docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + # cannot use docker run -v because the docker executor does not support volumes + # docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml + id=`docker create ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=core.xml` + docker start -a $id + docker cp $id:/build/core/core.xml test-results/ + docker rm $id - store_test_results: path: test-results/core.xml deploy: From 4bbe80824721a57cc6805a31345f51c9e49d3c52 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:34:16 +0200 Subject: [PATCH 34/37] CI/CD: remove temporary images on Dockerhub (trying to catch both PR and master, both success and failure) --- .circleci/config.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3346e2..3d56444 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,6 +165,14 @@ jobs: steps: - dockerhub-load: tag: minim<< parameters.variant >> + - unless: + condition: + equal: [ master, << pipeline.git.branch >>] + # only PR testing โ€“ not needed anymore + steps: + - dockerhub-remove: + tag: minim<< parameters.variant >> + when: always - build: dep: minim<< parameters.variant >> tag: medi<< parameters.variant >> @@ -180,11 +188,24 @@ jobs: steps: - dockerhub-load: tag: medi<< parameters.variant >> + - unless: + condition: + equal: [ master, << pipeline.git.branch >>] + # only PR testing โ€“ not needed anymore + steps: + - dockerhub-remove: + tag: medi<< parameters.variant >> + when: always - build: dep: medi<< parameters.variant >> tag: maxim<< parameters.variant >> - - dockerhub-store: - tag: maxim<< parameters.variant >> + - when: + condition: + equal: [ master, << pipeline.git.branch >>] + # only PR testing โ€“ not needed + steps: + - dockerhub-store: + tag: maxim<< parameters.variant >> deploy-mini: docker: - image: cimg/base:current-22.04 @@ -199,6 +220,7 @@ jobs: tag: minim<< parameters.variant >> - dockerhub-remove: tag: minim<< parameters.variant >> + when: always deploy-medi: docker: - image: cimg/base:current-22.04 @@ -213,6 +235,7 @@ jobs: tag: medi<< parameters.variant >> - dockerhub-remove: tag: medi<< parameters.variant >> + when: always deploy-maxi: docker: - image: cimg/base:current-22.04 @@ -227,6 +250,7 @@ jobs: tag: maxim<< parameters.variant >> - dockerhub-remove: tag: maxim<< parameters.variant >> + when: always workflows: version: 2 From cd8333fa1f4f4a7b91e3713f9dd9569bbfe3efde Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:59:57 +0200 Subject: [PATCH 35/37] CI/CD: move when-attribute to step definition --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d56444..2e6fb7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,12 @@ commands: string steps: - run: + name: use Dockerhub API to remove temporary image command: | HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/ + # must be unconditional (both on_error and on_success) + when: always build: description: "template for all docker-* build jobs" parameters: @@ -172,7 +175,6 @@ jobs: steps: - dockerhub-remove: tag: minim<< parameters.variant >> - when: always - build: dep: minim<< parameters.variant >> tag: medi<< parameters.variant >> @@ -195,7 +197,6 @@ jobs: steps: - dockerhub-remove: tag: medi<< parameters.variant >> - when: always - build: dep: medi<< parameters.variant >> tag: maxim<< parameters.variant >> @@ -220,7 +221,6 @@ jobs: tag: minim<< parameters.variant >> - dockerhub-remove: tag: minim<< parameters.variant >> - when: always deploy-medi: docker: - image: cimg/base:current-22.04 @@ -235,7 +235,6 @@ jobs: tag: medi<< parameters.variant >> - dockerhub-remove: tag: medi<< parameters.variant >> - when: always deploy-maxi: docker: - image: cimg/base:current-22.04 @@ -250,7 +249,6 @@ jobs: tag: maxim<< parameters.variant >> - dockerhub-remove: tag: maxim<< parameters.variant >> - when: always workflows: version: 2 From af6e0054d38f73b9e6d2586c66864cfa4e333d32 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:28:11 +0200 Subject: [PATCH 36/37] CI/CD: fix Dockerhub API password variable name --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e6fb7f..51c9e7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,8 @@ commands: - run: name: use Dockerhub API to remove temporary image command: | - HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASSWORD\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) + HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASS\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) + set -x curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/ # must be unconditional (both on_error and on_success) when: always From e33ac77359f9bbd76930c9a850299cb4f0cc1977 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 15 Oct 2024 16:32:14 +0000 Subject: [PATCH 37/37] :memo: update changelog --- CHANGELOG.md | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ca1328..170b52a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,20 @@ # Changelog -## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) - -### [ocrd_fileformat](https://github.com/OCR-D/ocrd_fileformat) [ff46bd7](https://github.com/OCR-D/ocrd_fileformat/commits/ff46bd7)..[8ab078d](https://github.com/OCR-D/ocrd_fileformat/commits/8ab078d) - -> Release: [v0.11.1](https://github.com/OCR-D/ocrd_fileformat/releases/v0.11.1) - - -## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) - -### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[db65d7f](https://github.com/cisocrgroup/ocrd_cis/commits/db65d7f) - -> Release: [v0.1.5](https://github.com/cisocrgroup/ocrd_cis/releases/v0.1.5) - - ## Unreleased -## [v2024-10-11](https://github.com/OCR-D/ocrd_all/releases/v2024-10-11) +## [v2024-10-15](https://github.com/OCR-D/ocrd_all/releases/v2024-10-15) +Changes: + + - added new module ocrd_page2alto (also in ocrd_fileformat, now with standalone processor) + - new fixup recipes for shared `venv` without dependency conflicts + - protect `venv` creation by semaphore as well + - docker: update OCRD_MODULES (default selection for custom `make docker`) + - docker: fix `minimum` and `medium` module lists + - docker: do not rm `venv` created by previous stage + - CI/CD: rewrite CircleCI config to split up mini/medi/maxi into interdependent incremental jobs + - CI/CD: fix storing test results + ### [core](https://github.com/OCR-D/core) [92b217e](https://github.com/OCR-D/core/commits/92b217e)..[85bde15](https://github.com/OCR-D/core/commits/85bde15) > Release: [v2.70.0](https://github.com/OCR-D/core/releases/v2.70.0) @@ -135,10 +132,11 @@ > * Merge branch 'master' into resolve-1257 > * load tool json locally -### [dinglehopper](https://github.com/qurator-spk/dinglehopper) [129e6eb](https://github.com/qurator-spk/dinglehopper/commits/129e6eb)..[2e6fe0c](https://github.com/qurator-spk/dinglehopper/commits/2e6fe0c) +### [dinglehopper](https://github.com/qurator-spk/dinglehopper) [129e6eb](https://github.com/qurator-spk/dinglehopper/commits/129e6eb)..[071e6a8](https://github.com/qurator-spk/dinglehopper/commits/071e6a8) > Release: [v0.9.7](https://github.com/qurator-spk/dinglehopper/releases/v0.9.7) + > * Merge pull request #120 from joschrew/dockerfile > * Merge pull request #113 from qurator-spk/python-3.13 > * โœ” pre-commit: Add license check > * ๐Ÿ› Fix --version option in OCR-D CLI @@ -220,7 +218,7 @@ Submodule eynollah 032a99e...51f6ef6: > * ๐Ÿงน Update OCR-D API usage w.r.t. workspace.add_file() > * โš™ pre-commit: Update hooks -### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[fe122ae](https://github.com/cisocrgroup/ocrd_cis/commits/fe122ae) +### [ocrd_cis](https://github.com/cisocrgroup/ocrd_cis) [38ce45b](https://github.com/cisocrgroup/ocrd_cis/commits/38ce45b)..[db65d7f](https://github.com/cisocrgroup/ocrd_cis/commits/db65d7f) > Release: [v0.1.5](https://github.com/cisocrgroup/ocrd_cis/releases/v0.1.5) @@ -257,7 +255,7 @@ Submodule eynollah 032a99e...51f6ef6: > * add Dockerhub CD > * Merge pull request #5 from joschrew/dockerfile -### [ocrd_fileformat](https://github.com/OCR-D/ocrd_fileformat) [ff46bd7](https://github.com/OCR-D/ocrd_fileformat/commits/ff46bd7)..[60cb696](https://github.com/OCR-D/ocrd_fileformat/commits/60cb696) +### [ocrd_fileformat](https://github.com/OCR-D/ocrd_fileformat) [ff46bd7](https://github.com/OCR-D/ocrd_fileformat/commits/ff46bd7)..[8ab078d](https://github.com/OCR-D/ocrd_fileformat/commits/8ab078d) > Release: [v0.11.1](https://github.com/OCR-D/ocrd_fileformat/releases/v0.11.1) @@ -325,10 +323,18 @@ Submodule eynollah 032a99e...51f6ef6: > * Merge pull request #1015 from Gornoka:patch-1 -### [sbb_binarization](https://github.com/qurator-spk/sbb_binarization) [978f425](https://github.com/qurator-spk/sbb_binarization/commits/978f425)..[ccfc821](https://github.com/qurator-spk/sbb_binarization/commits/ccfc821) +### [sbb_binarization](https://github.com/qurator-spk/sbb_binarization) [978f425](https://github.com/qurator-spk/sbb_binarization/commits/978f425)..[d259795](https://github.com/qurator-spk/sbb_binarization/commits/d259795) > Release: [v0.1.0](https://github.com/qurator-spk/sbb_binarization/releases/v0.1.0) + > * docker: rebase on core-cuda stage + > * forgot to include package data + > * dockerfile: switch to pyproject.toml + > * relax TF requirement + > * CI: remove py37 from matrix + > * remove setup.py + > * add pyproject.toml + > * make docker: fix docker tag > * remove shebang from setup.py (somehow breaking py39) > * CI: increase memory on VM > * make install: update setuptools, too