Merge pull request #452 from OCR-D/update-2024-10-11

Update 2024 10 11
OCR-D · Oct 15, 2024 · 3e33522 · 3e33522
2 parents 25fa279 + e33ac77
commit 3e33522
Show file tree

Hide file tree

Showing 25 changed files with 645 additions and 92 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,78 +1,109 @@
 version: 2.1
-jobs:
+
+commands:
+  dockerhub-store:
+    description: "store an image on Dockerhub"
+    parameters:
+      tag:
+        description: "image tag (full variant name) to be saved"
+        type:
+          string
+    steps:
+      - run:
+          name: Login to Docker Hub
+          command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
+      - run:
+          name: save image
+          command: |
+            docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
+            docker push ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
+          no_output_timeout: 30m
+  dockerhub-load:
+    description: "load an image from Dockerhub"
+    parameters:
+      tag:
+        description: "image tag (full variant name) to be pulled"
+        type:
+          string
+    steps:
+      - run:
+          name: load previous image
+          command: |
+            docker pull ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
+            docker tag ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} ocrd/all:<< parameters.tag >>
+  dockerhub-remove:
+    description: "remove an image on Dockerhub"
+    parameters:
+      tag:
+        description: "image tag (full variant name) to be removed"
+        type:
+          string
+    steps:
+      - run:
+          name: use Dockerhub API to remove temporary image
+          command: |
+            HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASS\"}" https://hub.docker.com/v2/users/login/ | jq -r .token)
+            set -x
+            curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/
+          # must be unconditional (both on_error and on_success)
+          when: always
   build:
-    docker:
-      - image: cimg/base:current-22.04
-    resource_class: large
+    description: "template for all docker-* build jobs"
+    parameters:
+      tag:
+        description: "image tag (full variant name) to be built"
+        type:
+          string
+      dep:
+        description: "image tag (full variant name) this depends on (should _not_ be rebuilt)"
+        type:
+          string
+        default: "core"
     steps:
       - checkout
-      - setup_remote_docker      # https://circleci.com/docs/2.0/building-docker-images/
+      - setup_remote_docker
       - run:
-          name: build image
-          command: make docker-maximum-cuda GIT_DEPTH=--single-branch DOCKER_PARALLEL=-j4
+          name: Build Docker image
+          command: make docker-<< parameters.tag >> -o docker-<< parameters.dep >> GIT_DEPTH=--single-branch
           no_output_timeout: 30m
       - run:
           name: test image
           command: |
             mkdir test-results
-            docker run --rm -v $PWD:/data ocrd/all:maximum-cuda make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml
+            # cannot use docker run -v because the docker executor does not support volumes
+            # docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml
+            id=`docker create ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=core.xml`
+            docker start -a $id
+            docker cp $id:/build/core/core.xml test-results/
+            docker rm $id
       - store_test_results:
-          path: test-results
-      - when:
-          # takes too long for 1h1m CircleCI timeout overall
-          # also, storage is limited...
-          condition: false
-          steps:
-            - run:
-                name: persist image
-                command: |
-                  sudo apt install pigz
-                  docker image save ocrd/all:maximum-cuda | pigz --fast > ocrd-all-maximum.tar.gz
-                no_output_timeout: 30m
-            # can be downloaded from CircleCI.com and imported via "docker image load"
-            - store_artifacts:
-                path: ocrd-all-maximum.tar.gz
-                destination: artifacts
+          path: test-results/core.xml
   deploy:
-    docker:
-      - image: cimg/base:current-22.04
+    description: "template for all docker-* deploy jobs"
     parameters:
-      variant:
+      tag:
+        description: "image tag (full variant name) to be pushed"
         type:
           string
     steps:
       - checkout
-      - setup_remote_docker      # https://circleci.com/docs/2.0/building-docker-images/
+      - setup_remote_docker
       - run:
-          name: Build Docker image
-          command: make docker-<< parameters.variant >> GIT_DEPTH=--single-branch
-          # fails due to pip races: DOCKER_PARALLEL=-j3
-          no_output_timeout: 30m
-      - run:
-          name: Alias Docker images
-          command: docker tag ocrd/all:<< parameters.variant >> ocrd/all:<< parameters.variant >>-git
+          name: Alias Docker image
+          command: docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-git
       - run:
           name: Login to Docker Hub
           command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
       - run:
           name: Push images to Docker Hub
           no_output_timeout: 2.5h
           command: |
-            docker push ocrd/all:<< parameters.variant >>
-            docker push ocrd/all:<< parameters.variant >>-git
+            docker push ocrd/all:<< parameters.tag >>
+            docker push ocrd/all:<< parameters.tag >>-git
       - when:
           condition:
-            equal: [ maximum, << parameters.variant >> ]
+            equal: [ maximum, << parameters.tag >> ]
           steps:
-            - run:
-                name: Alias and push intermediate variants
-                command: |
-                  docker tag ocrd/all:medium ocrd/all:medium-git
-                  docker tag ocrd/all:minimum ocrd/all:minimum-git
-                  docker push ocrd/all:minimum
-                  docker push ocrd/all:minimum-git
-                  docker push ocrd/all:medium
-                  docker push ocrd/all:medium-git
             - run:
                 name: store ocrd-all-tool.json
                 command: |
@@ -91,28 +122,186 @@ jobs:
                 destination: ocrd-all-module-dir.json
       - when:
           condition:
-            equal: [ maximum-cuda, << parameters.variant >> ]
+            equal: [ maximum-cuda, << parameters.tag >> ]
           steps:
             - run:
                 name: Create a date-versioned mirror of ocrd/all:maximum-cuda
                 command: bash release.sh release-dockerhub
             - run:
                 name: Update badge
                 command: curl -X POST "$MICROBADGER_WEBHOOK" || true
+      - when:
+          # takes too long for 1h1m CircleCI timeout overall
+          # also, storage is limited...
+          condition: false
+          steps:
+            - run:
+                name: persist image
+                command: |
+                  sudo apt install pigz
+                  docker image save ocrd/all:<< parameters.tag >> | pigz --fast > ocrd-all-maximum.tar.gz
+                no_output_timeout: 30m
+            # can be downloaded from CircleCI.com and imported via "docker image load"
+            - store_artifacts:
+                path: ocrd-all-maximum.tar.gz
+                destination: artifacts
+
+jobs:
+  build-mini:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - build:
+          tag: minim<< parameters.variant >>
+      - dockerhub-store:
+          tag: minim<< parameters.variant >>
+  build-medi:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - dockerhub-load:
+          tag: minim<< parameters.variant >>
+      - unless:
+          condition: 
+            equal: [ master, << pipeline.git.branch >>]
+          # only PR testing – not needed anymore
+          steps:
+            - dockerhub-remove:
+                tag: minim<< parameters.variant >>
+      - build:
+          dep: minim<< parameters.variant >>
+          tag: medi<< parameters.variant >>
+      - dockerhub-store:
+          tag: medi<< parameters.variant >>
+  build-maxi:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - dockerhub-load:
+          tag: medi<< parameters.variant >>
+      - unless:
+          condition: 
+            equal: [ master, << pipeline.git.branch >>]
+          # only PR testing – not needed anymore
+          steps:
+            - dockerhub-remove:
+                tag: medi<< parameters.variant >>
+      - build:
+          dep: medi<< parameters.variant >>
+          tag: maxim<< parameters.variant >>
+      - when:
+          condition: 
+            equal: [ master, << pipeline.git.branch >>]
+          # only PR testing – not needed
+          steps:
+            - dockerhub-store:
+                tag: maxim<< parameters.variant >>
+  deploy-mini:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - dockerhub-load:
+          tag: minim<< parameters.variant >>
+      - deploy:
+          tag: minim<< parameters.variant >>
+      - dockerhub-remove:
+          tag: minim<< parameters.variant >>
+  deploy-medi:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - dockerhub-load:
+          tag: medi<< parameters.variant >>
+      - deploy:
+          tag: medi<< parameters.variant >>
+      - dockerhub-remove:
+          tag: medi<< parameters.variant >>
+  deploy-maxi:
+    docker:
+      - image: cimg/base:current-22.04
+    parameters:
+      variant:
+        type:
+          string
+    steps:
+      - dockerhub-load:
+          tag: maxim<< parameters.variant >>
+      - deploy:
+          tag: maxim<< parameters.variant >>
+      - dockerhub-remove:
+          tag: maxim<< parameters.variant >>
+
 workflows:
   version: 2
-  build-master:
+  build:
     jobs:
-      - deploy:
+      - build-mini:
+          name: build-minim<< matrix.variant >>
+          matrix:
+            parameters:
+              variant: [um, um-cuda]
+      - build-medi:
+          name: build-medi<< matrix.variant >>
+          matrix:
+            parameters:
+              variant: [um, um-cuda]
+          requires:
+            - build-minim<< matrix.variant >>
+      - build-maxi:
+          name: build-maxim<< matrix.variant >>
           matrix:
             parameters:
-              variant: [maximum, maximum-cuda]
+              variant: [um, um-cuda]
+          requires:
+            - build-medi<< matrix.variant >>
+      - deploy-mini:
+          name: deploy-minim<< matrix.variant >>
+          matrix:
+            parameters:
+              variant: [um, um-cuda]
+          requires:
+            - build-minim<< matrix.variant >>
           filters:
             branches:
               only: master
-  build-pull-request:
-    jobs:
-      - build:
+      - deploy-medi:
+          name: deploy-medi<< matrix.variant >>
+          matrix:
+            parameters:
+              variant: [um, um-cuda]
+          requires:
+            - build-medi<< matrix.variant >>
+          filters:
+            branches:
+              only: master
+      - deploy-maxi:
+          name: deploy-maxim<< matrix.variant >>
+          matrix:
+            parameters:
+              variant: [um, um-cuda]
+          requires:
+            - build-maxim<< matrix.variant >>
           filters:
             branches:
-              ignore: master
+              only: master
+
diff --git a/.gitmodules b/.gitmodules
@@ -94,3 +94,6 @@
 [submodule "ocrd_froc"]
 	path = ocrd_froc
 	url = https://github.com/OCR-D/ocrd_froc
+[submodule "ocrd_page2alto"]
+	path = ocrd_page2alto
+	url = https://github.com/OCR-D/page-to-alto