Merge branch 'test/setup-gitlab-cicd' into 'develop'

Add gitlab ci/cd Closes #807 See merge request sds-dev/sd-submit/metadata-submitter!809
CSCfi · Jan 11, 2024 · da4a785 · da4a785
2 parents 2930683 + f039f6c
commit da4a785
Show file tree

Hide file tree

Showing 12 changed files with 107 additions and 33 deletions.
diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml
@@ -44,7 +44,7 @@ jobs:
         with:
           context: .
           push: true
-          file: ./Dockerfile
+          file: ./dockerfiles/Dockerfile
           tags: localhost:5000/metadata-submitter:latest
           cache-from: localhost:5000/metadata-submitter:latest
           cache-to: type=local,dest=/tmp/.buildx-cache

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -40,7 +40,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           context: .
-          file: ./Dockerfile
+          file: ./dockerfiles/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
           build-args: |
             BRANCH=${{ github.ref_name == 'master' && 'master' || 'develop' }}

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,77 @@
+stages:
+  - verify
+  - test
+
+variables:
+  BUILD_IMAGE: $ARTIFACTORY_SERVER/sds/sdd-common-ci
+
+linting:
+  stage: verify
+  tags:
+    - docker-exec
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+  image: $BUILD_IMAGE
+  before_script:
+    - pip install -r requirements-dev.txt
+  script:
+    - pre-commit run --all-files -c .pre-commit-config.yaml --show-diff-on-failure --color never
+    - tox -e docs
+
+spellcheck:
+  stage: verify
+  tags:
+    - docker-exec
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+  image: $BUILD_IMAGE
+  before_script:
+    - pip install pyspelling
+  script:
+    - pyspelling
+
+unittest:
+  stage: test
+  tags:
+    - docker-exec
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+  image: $BUILD_IMAGE
+  before_script:
+    - pip install -r requirements-dev.txt
+  script:
+    - tox -e py311
+    - coverage report --precision=1
+  coverage: '/TOTAL.*\s+(\d+\%)/'
+
+# TODO: make docker compose work in gitlab runner
+# integration:
+#   stage: verify
+#   tags:
+#     - docker-exec
+#   rules:
+#     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+#   services:
+#     - name: registry.hub.docker.com/library/docker:24.0.6-dind
+#       alias: docker
+#   variables:
+#     DOCKER_HOST: tcp://docker:2375/
+#     DOCKER_DRIVER: overlay2
+#     BASE_URL: http://localhost:5430
+#     OIDC_URL: http://localhost:8000
+#     OIDC_URL_TEST: http://mockauth:8000
+#     FF_NETWORK_PER_BUILD: "true"
+#   image: docker:24.0.6
+#   before_script:
+#     - apk --update --no-cache add python3
+#     - python3 -m ensurepip
+#     - pip3 install --no-cache --upgrade pip setuptools
+#     - pip install aiohttp aiofiles motor pytest pytest-asyncio
+#   script:
+#     - docker compose -f docker-compose.yml --env-file .env.example up --build -d
+#     - sleep 30
+#     - docker network inspect metadata-submitter_default
+#     - docker compose -f docker-compose.yml --env-file .env.example ps
+#     - pytest tests/integration/
+#   after_script:
+#     - docker compose down -v
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,7 +29,7 @@ repos:
       exclude: tests/test_files/study/SRP000539_invalid[0-9]?.xml
 
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.10.0
     hooks:
     - id: black
       args: [-l, "120"]
@@ -41,24 +41,13 @@ repos:
       args: [--profile, black, --filter-files]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     - id: flake8
       additional_dependencies:
         - flake8-docstrings
         - flake8-annotations
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.4.1
-    hooks:
-    - id: mypy
-      args: [--ignore-missing-imports, metadata_backend/]
-      pass_filenames: false
-      additional_dependencies:
-        - types-python-dateutil
-        - types-ujson
-        - types-requests
-
   - repo:  https://github.com/PyCQA/bandit
     rev: 1.7.5
     hooks:
@@ -73,7 +62,7 @@ repos:
       files: docs/specification.yml
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.23.3
+    rev: 0.27.0
     hooks:
       - id: check-metaschema
         name: Validate schemas
@@ -88,7 +77,7 @@ repos:
       - id: check-github-workflows
 
   - repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.7'
+    rev: 'v2.10'
     hooks:
     - id: vulture
       args: [metadata_backend/, whitelist_dead_code.py]
@@ -115,6 +104,13 @@ repos:
           -sn, # Don't display the score
         ]
 
+    - id: mypy
+      name: mypy
+      entry: mypy
+      language: system
+      types: [python]
+      files: ^metadata_backend
+
     - id: pyspelling-docs
       name: pyspelling-docs
       entry: ./scripts/dictionary/pyspelling.sh

diff --git a/README.md b/README.md
@@ -137,7 +137,7 @@ If it doesn't work right away, check your settings in `.env` and restart the ser
 
 Production version can be built and run with following docker commands:
 ```bash
-$ docker build --no-cache . -t metadata-submitter
+$ docker build --no-cache -f dockerfiles/Dockerfile -t cscfi/metadata-submitter .
 $ docker run -p 5430:5430 metadata-submitter
 ```
 

diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml
@@ -14,7 +14,7 @@ services:
       - ./config:/origcerts
   backend:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -103,7 +103,7 @@ services:
       - ./scripts/rabbitmq/rabbitmq-definitions.json:/etc/rabbitmq/rabbitmq-definitions.json
   mockauth:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -121,7 +121,7 @@ services:
     entrypoint: [ "python", "/mock_auth.py", "0.0.0.0", "8000" ]
   mockdoi:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -135,7 +135,7 @@ services:
     entrypoint: [ "python", "/mock_doi_api.py", "0.0.0.0", "8001" ]
   mockmetax:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -149,7 +149,7 @@ services:
     entrypoint: [ "python", "/mock_metax_api.py", "0.0.0.0", "8002" ]
   mockrems:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.4"
 services:
   backend:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -79,7 +79,7 @@ services:
       - ./scripts/rabbitmq/rabbitmq-definitions.json:/etc/rabbitmq/rabbitmq-definitions.json
   mockauth:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -97,7 +97,7 @@ services:
     entrypoint: [ "python", "/mock_auth.py", "0.0.0.0", "8000" ]
   mockdoi:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -111,7 +111,7 @@ services:
     entrypoint: [ "python", "/mock_doi_api.py", "0.0.0.0", "8001" ]
   mockmetax:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev
@@ -125,7 +125,7 @@ services:
     entrypoint: [ "python", "/mock_metax_api.py", "0.0.0.0", "8002" ]
   mockrems:
     build:
-      dockerfile: Dockerfile-dev
+      dockerfile: ./dockerfiles/Dockerfile-dev
       context: .
       target: develop
     image: cscfi/metadata-submitter-dev

diff --git a/Dockerfile → dockerfiles/Dockerfile b/Dockerfile → dockerfiles/Dockerfile
diff --git a/Dockerfile-dev → dockerfiles/Dockerfile-dev b/Dockerfile-dev → dockerfiles/Dockerfile-dev
@@ -29,8 +29,8 @@ CMD ["metadata_submitter"]
 FROM appbase as local
 #=======================
 
-COPY requirements-dev.txt requirements.txt
+COPY requirements-dev.txt .
 
-RUN pip install --no-cache-dir
+RUN pip install --no-cache-dir -r requirements-dev.txt
 
 ENV PYTHONUNBUFFERED=1
diff --git a/docs/deploy.rst b/docs/deploy.rst
@@ -57,7 +57,7 @@ The production image can be built and run with following docker commands:
 
 .. code-block:: console
 
-    docker build --no-cache . -t cscfi/metadata-submitter
+    docker build --no-cache -f dockerfiles/Dockerfile -t cscfi/metadata-submitter .
     docker run -p 5430:5430 cscfi/metadata-submitter
 
 .. important:: Requires running MongoDB and consider setting the environment variables as pointed out in :ref:`backend`.

diff --git a/docs/dictionary/wordlist.txt b/docs/dictionary/wordlist.txt
@@ -189,6 +189,7 @@ devcontainer
 divehi
 dnase
 dockerfile
+dockerfiles
 docstrings
 doi
 doiinfo
@@ -686,9 +687,9 @@ subjectsschema
 submissionid
 submissionslice
 submissiontype
+submitter's
 submitterdemultiplexed
 submitterid
-submitter's
 submitters
 svg
 swati

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -193,7 +193,7 @@ multidict==6.0.4
     #   -r requirements.txt
     #   aiohttp
     #   yarl
-mypy==1.6.0
+mypy==1.6.1
     # via -r requirements-dev.in
 mypy-extensions==1.0.0
     # via