♻️ Add common library (#6495)

.github/CODEOWNERS

@@ -13,6 +13,7 @@ Makefile                                  @pcrespov @sanderegg
 /api/                                     @sanderegg @pcrespov @matusdrobuliak66
 /ci/                                      @sanderegg @pcrespov
 /docs/                                    @pcrespov
+/packages/common-library/                 @giancarloromeo
 /packages/models-library/                 @sanderegg @pcrespov @matusdrobuliak66
 /packages/postgres-database/              @matusdrobuliak66
 /packages/pytest-simcore/                 @pcrespov @sanderegg

.github/workflows/ci-testing-deploy.yml

@@ -54,6 +54,7 @@ jobs:
       aws-library: ${{ steps.filter.outputs.aws-library }}
       dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
       models-library: ${{ steps.filter.outputs.models-library }}
+      common-library: ${{ steps.filter.outputs.common-library }}
       notifications-library: ${{ steps.filter.outputs.notifications-library }}
       postgres-database: ${{ steps.filter.outputs.postgres-database }}
       service-integration: ${{ steps.filter.outputs.service-integration }}
@@ -110,6 +111,8 @@ jobs:
               - 'services/docker-compose*'
               - 'scripts/mypy/*'
               - 'mypy.ini'
+            common-library:
+              - 'packages/common-library/**'
             notifications-library:
               - 'packages/notifications-library/**'
               - 'packages/postgres-database/**'
@@ -1593,6 +1596,47 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-common-library:
+    needs: changes
+    if: ${{ needs.changes.outputs.common-library == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] common-library"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-22.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: install uv
+        uses: yezz123/setup-uv@v4
+      - uses: actions/cache@v4
+        id: cache-uv
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/common-library.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/common-library.bash typecheck
+      - name: test
+        run: ./ci/github/unit-testing/common-library.bash test
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-notifications-library:
     needs: changes
     if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
@@ -1704,6 +1748,7 @@ jobs:
         unit-test-efs-guardian,
         unit-test-frontend,
         unit-test-models-library,
+        unit-test-common-library,
         unit-test-notifications-library,
         unit-test-osparc-gateway-server,
         unit-test-payments,

ci/github/unit-testing/common-library.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/common-library
+  make install-ci
+  popd
+  uv pip list
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/common-library
+  make tests-ci
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd packages/common-library
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/aws-library/requirements/_base.in

@@ -2,6 +2,7 @@
 # Specifies third-party dependencies for 'aws-library'
 #
 --constraint ../../../requirements/constraints.txt
+--requirement ../../../packages/common-library/requirements/_base.in
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in

packages/common-library/Makefile

@@ -0,0 +1,49 @@
+#
+# Targets for DEVELOPMENT of common Library
+#
+include ../../scripts/common.Makefile
+include ../../scripts/common-package.Makefile
+
+.PHONY: requirements
+requirements: ## compiles pip requirements (.in -> .txt)
+	@$(MAKE_C) requirements reqs
+
+
+.PHONY: install-dev install-prod install-ci
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	@uv pip sync requirements/$(subst install-,,$@).txt
+
+
+.PHONY: tests tests-ci
+tests: ## runs unit tests
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov=common_library \
+		--durations=10 \
+		--exitfirst \
+		--failed-first \
+		--pdb \
+		-vv \
+		$(CURDIR)/tests
+
+tests-ci: ## runs unit tests [ci-mode]
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-append \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov-report=xml \
+		--cov=common_library \
+		--durations=10 \
+		--log-date-format="%Y-%m-%d %H:%M:%S" \
+    --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --verbose \
+    -m "not heavy_load" \
+		$(CURDIR)/tests

packages/common-library/README.md

@@ -0,0 +1,42 @@
+# simcore pydantic common library
+
+Contains the common classes, functions and in general utilities for use in the simcore platform.
+
+## Installation
+
+```console
+make help
+make install-dev
+```
+
+## Test
+
+```console
+make help
+make test-dev
+```
+
+
+## Diagnostics
+
+How run diagnostics on the service metadata published in a docker registry?
+
+1. Setup environment
+```bash
+make devenv
+source .venv/bin/activate
+
+cd packages/common-library
+make install-dev
+```
+2. Set ``REGISTRY_*`` env vars in ``.env`` (in the repository base folder)
+3. Download test data, run diagnostics, archive tests-data, and cleanup
+```bash
+export DEPLOY_NAME=my-deploy
+
+make pull_test_data >$DEPLOY_NAME-registry-diagnostics.log 2>&1
+pytest -vv -m diagnostics >>$DEPLOY_NAME-registry-diagnostics.log 2>&1
+zip -r $DEPLOY_NAME-registry-test-data.zip tests/data/.downloaded-ignore
+rm -r tests/data/.downloaded-ignore
+```
+4. Move all ``$DEPLOY_NAME-*`` files to an archive

packages/common-library/VERSION

@@ -0,0 +1 @@
+0.1.0

packages/common-library/requirements/Makefile

@@ -0,0 +1,6 @@
+#
+# Targets to pip-compile requirements
+#
+include ../../../requirements/base.Makefile
+
+# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt

packages/common-library/requirements/_base.in

@@ -0,0 +1,6 @@
+#
+# Specifies third-party dependencies for 'common-library'
+#
+--constraint ../../../requirements/constraints.txt
+
+pydantic

packages/common-library/requirements/_base.txt

@@ -0,0 +1,12 @@
+annotated-types==0.7.0
+    # via pydantic
+pydantic==2.9.2
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/_base.in
+pydantic-core==2.23.4
+    # via pydantic
+typing-extensions==4.12.2
+    # via
+    #   pydantic
+    #   pydantic-core

packages/common-library/requirements/_test.in

@@ -0,0 +1,21 @@
+#
+# Specifies dependencies required to run 'common-library'
+#
+--constraint ../../../requirements/constraints.txt
+
+# Adds base AS CONSTRAINT specs, not requirement.
+#  - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt
+#
+--constraint _base.txt
+
+coverage
+faker
+pytest
+pytest-asyncio
+pytest-cov
+pytest-icdiff
+pytest-instafail
+pytest-mock
+pytest-runner
+pytest-sugar
+python-dotenv

packages/common-library/requirements/_test.txt

@@ -0,0 +1,55 @@
+coverage==7.6.1
+    # via
+    #   -r requirements/_test.in
+    #   pytest-cov
+faker==30.1.0
+    # via -r requirements/_test.in
+icdiff==2.0.7
+    # via pytest-icdiff
+iniconfig==2.0.0
+    # via pytest
+packaging==24.1
+    # via
+    #   pytest
+    #   pytest-sugar
+pluggy==1.5.0
+    # via pytest
+pprintpp==0.4.0
+    # via pytest-icdiff
+pytest==8.3.3
+    # via
+    #   -r requirements/_test.in
+    #   pytest-asyncio
+    #   pytest-cov
+    #   pytest-icdiff
+    #   pytest-instafail
+    #   pytest-mock
+    #   pytest-sugar
+pytest-asyncio==0.23.8
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/_test.in
+pytest-cov==5.0.0
+    # via -r requirements/_test.in
+pytest-icdiff==0.9
+    # via -r requirements/_test.in
+pytest-instafail==0.5.0
+    # via -r requirements/_test.in
+pytest-mock==3.14.0
+    # via -r requirements/_test.in
+pytest-runner==6.0.1
+    # via -r requirements/_test.in
+pytest-sugar==1.0.0
+    # via -r requirements/_test.in
+python-dateutil==2.9.0.post0
+    # via faker
+python-dotenv==1.0.1
+    # via -r requirements/_test.in
+six==1.16.0
+    # via python-dateutil
+termcolor==2.5.0
+    # via pytest-sugar
+typing-extensions==4.12.2
+    # via
+    #   -c requirements/_base.txt
+    #   faker

packages/common-library/requirements/_tools.in

@@ -0,0 +1,5 @@
+--constraint ../../../requirements/constraints.txt
+--constraint _base.txt
+--constraint _test.txt
+
+--requirement ../../../requirements/devenv.txt