Merge branch 'master' into upgrade-api-server-dependencies

bisgaard-itis · Dec 6, 2024 · 05484a3 · 05484a3
2 parents 8c9baf8 + 35f11df
commit 05484a3
Show file tree

Hide file tree

Showing 23 changed files with 452 additions and 1,783 deletions.
diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml
@@ -2655,6 +2655,8 @@ jobs:
         system-test-environment-setup,
         system-test-public-api,
         system-test-swarm-deploy,
+        system-api-specs,
+        system-backwards-compatibility
       ]
     runs-on: ubuntu-latest
     steps:
@@ -2719,3 +2721,78 @@ jobs:
         env:
           TAG_PREFIX: hotfix-staging-github
         run: ./ci/deploy/dockerhub-deploy.bash -n
+
+  system-api-specs:
+    needs: [changes]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 10
+    name: "[sys] check api-specs are up to date"
+    runs-on: ubuntu-latest
+    steps:
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.4.x"
+          enable-cache: false
+      - name: checkout source branch
+        uses: actions/checkout@v4
+      - name: Regenerate specs and check
+        run: |
+          uv venv .venv && source .venv/bin/activate
+          make openapi-specs
+          ./ci/github/helpers/openapi-specs-diff.bash diff \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/refs/heads/${{ github.event.pull_request.head.ref }} \
+            .
+
+  system-backwards-compatibility:
+    needs: [changes, system-api-specs]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 10
+    name: "[sys] api-server backwards compatibility"
+    runs-on: ubuntu-latest
+    steps:
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.4.x"
+          enable-cache: false
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: check api-server backwards compatibility
+        run: |
+          ./scripts/openapi-diff.bash breaking --fail-on ERR\
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }}/services/api-server/openapi.json \
+            /specs/services/api-server/openapi.json
+
+  api-spec-backwards-compatibility:
+    needs: [changes, system-api-specs]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
+    continue-on-error: true
+    timeout-minutes: 10
+    name: "api-specs-backwards-compatibility"
+    runs-on: ubuntu-latest
+    steps:
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.4.x"
+          enable-cache: false
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Check openapi-specs backwards compatibility
+        run: |
+          ./ci/github/helpers/openapi-specs-diff.bash breaking \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }} \
+            .
diff --git a/Makefile b/Makefile
@@ -10,7 +10,7 @@
 .DEFAULT_GOAL := help
 
 SHELL := /bin/bash
-
+.SHELLFLAGS := -o errexit -o pipefail -c
 MAKE_C := $(MAKE) --no-print-directory --directory
 
 # Operating system
@@ -84,7 +84,7 @@ export SWARM_STACK_NAME_NO_HYPHEN = $(subst -,_,$(SWARM_STACK_NAME))
 export DOCKER_IMAGE_TAG ?= latest
 export DOCKER_REGISTRY  ?= itisfoundation
 
-
+MAKEFILES_WITH_OPENAPI_SPECS := $(shell find . -mindepth 2 -type f -name 'Makefile' -not -path '*/.*' -exec grep -l '^openapi-specs:' {} \; | xargs realpath)
 
 get_my_ip := $(shell (hostname --all-ip-addresses || hostname -i) 2>/dev/null | cut --delimiter=" " --fields=1)
 
@@ -131,6 +131,12 @@ test_python_version: ## Check Python version, throw error if compilation would f
 	@.venv/bin/python ./scripts/test_python_version.py
 
 
+.PHONY: _check_venv_active
+_check_venv_active:
+	# Checking whether virtual environment was activated
+	@python3 -c "import sys; assert sys.base_prefix!=sys.prefix"
+
+
 ## DOCKER BUILD -------------------------------
 #
 # - all builds are immediatly tagged as 'local/{service}:${BUILD_TARGET}' where BUILD_TARGET='development', 'production', 'cache'
@@ -573,9 +579,13 @@ new-service: .venv ## Bakes a new project from cookiecutter-simcore-pyservice an
 
 
 .PHONY: openapi-specs
-openapi-specs: ## bundles and validates openapi specifications and schemas of ALL service's API
-	@$(MAKE_C) services/web/server $@
-	@$(MAKE_C) services/storage $@
+openapi-specs: .env _check_venv_active ## generates and validates openapi specifications and schemas of ALL service's API
+	@for makefile in $(MAKEFILES_WITH_OPENAPI_SPECS); do \
+		echo "Generating openapi-specs using $${makefile}"; \
+		$(MAKE_C) $$(dirname $${makefile}) install-dev; \
+		$(MAKE_C) $$(dirname $${makefile}) $@; \
+		printf "%0.s=" {1..100} && printf "\n"; \
+	done
 
 
 .PHONY: settings-schema.json