Bug/ci: must fail on error (#139)

* add pygeohash * add pytz * fail ci on error * add celery * remove pipefail * add jsonref * start * little sleep for agate * sh to bash * [[ to [ * test * refactor dc * exit on error
gisaia · Apr 30, 2024 · 3256d8a · 3256d8a
1 parent cda2f76
commit 3256d8a
Show file tree

Hide file tree

Showing 10 changed files with 124 additions and 92 deletions.
diff --git a/docker-compose-tests.yaml b/docker-compose-tests.yaml
@@ -16,9 +16,12 @@ services:
       - ServerOptions__HostName=smtp4dev
 
   arlas-server:
-    image: gisaia/arlas-server:${ARLAS_SERVER_VERSION:-23.0.7}
+    image: gisaia/arlas-server:${ARLAS_SERVER_VERSION:-24.1.2}
     platform: linux/amd64
     container_name: arlas-server
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
     networks:
       - aias
     ports:
@@ -35,7 +38,12 @@ services:
       - ARLAS_ELASTIC_INDEX=.arlas
       - ARLAS_APP_PATH=/
       - ARLAS_BASE_URI=http://localhost:9999/server/
-
+    healthcheck:
+          test: ["CMD","java","HttpHealthcheck.java","http://localhost:9999/admin/healthcheck"]
+          interval: 10s
+          timeout: 10s
+          retries: 6
+
   agate:
     networks:
       - aias
@@ -44,6 +52,9 @@ services:
       dockerfile: Dockerfile-agate
     image: agate:latest
     container_name: 'agate'
+    depends_on:
+      arlas-server:
+        condition: service_healthy
     environment:
       - AGATE_CORS_ORIGINS=${AGATE_CORS_ORIGINS:-*}
       - AGATE_CORS_METHODS=${AGATE_CORS_METHODS:-*}
@@ -59,7 +70,3 @@ services:
 #      - ASSET_MINIO_PUBLIC_PATTERN="(/collections/)(?P<collection>[^/]+)/items/(?P<item>[^/]+)/assets/thumbnail"
     ports:
       - $AGATE_PORT:$AGATE_PORT
-    depends_on:
-      - arlas-server
-
-
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -14,11 +14,17 @@ services:
       - "MINIO_ACCESS_KEY=airs"
       - "MINIO_SECRET_KEY=airssecret"
     command: server /export --console-address ":9001"
+    healthcheck:
+      test: "mc alias set myminio http://localhost:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; mc ping -x myminio"
+      interval: 5s
+      timeout: 10s
+      retries: 3
 
   createbuckets:
     image: minio/mc
     depends_on:
-      - minio
+      minio:
+        condition: service_healthy
     environment:
       - BUCKET_NAME=${BUCKET_NAME:-airstest}
       - LOGIN=${LOGIN:-airs}
@@ -47,6 +53,11 @@ services:
       - xpack.security.transport.ssl.enabled=false
     ports:
       - 9200:9200
+    healthcheck:
+      test: "curl -s -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
+      interval: 10s
+      timeout: 10s
+      retries: 24
 
   rabbitmq:
     image: rabbitmq:${RABBITMQ_VERSION:-3-management}
@@ -56,6 +67,11 @@ services:
     ports:
       - 15672:15672
       - 5672:5672
+    healthcheck:
+      test: rabbitmq-diagnostics check_port_connectivity
+      interval: 30s
+      timeout: 30s
+      retries: 10
 
   smtp4dev:
     image: rnwood/smtp4dev:linux-${PLATFORM}-3.2.0-ci20221023104
@@ -76,6 +92,50 @@ services:
     container_name: redis
     ports:
       - 6379:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+
+  airs-server:
+    networks:
+      - aias
+    build:
+      context: '.'
+      dockerfile: Dockerfile-airs
+    image: airs-server:latest
+    container_name: 'airs-server'
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    environment:
+      - AIRS_HOST=0.0.0.0
+      - AIRS_PORT=8000
+      - AIRS_CORS_ORIGINS=${AIRS_CORS_ORIGINS:-*}
+      - AIRS_CORS_METHODS=${AIRS_CORS_METHODS:-*}
+      - AIRS_CORS_HEADERS=${AIRS_CORS_HEADERS:-*}
+      - AIRS_INDEX_ENDPOINT_URL=${AIRS_INDEX_ENDPOINT_URL:-http://localhost:9200}
+      - AIRS_INDEX_COLLECTION_PREFIX=${AIRS_INDEX_COLLECTION_PREFIX:-airs}
+      - AIRS_INDEX_LOGIN=${AIRS_INDEX_LOGIN:-None}
+      - AIRS_INDEX_PWD=${AIRS_INDEX_PWD:-None}
+      - AIRS_S3_BUCKET=${AIRS_S3_BUCKET:-airstest}
+      - AIRS_S3_ACCESS_KEY_ID=${AIRS_S3_ACCESS_KEY_ID}
+      - AIRS_S3_SECRET_ACCESS_KEY=${AIRS_S3_SECRET_ACCESS_KEY}
+      - AIRS_S3_REGION=${AIRS_S3_REGION:-None}
+      - AIRS_S3_TIER=${AIRS_S3_TIER:-Standard}
+      - AIRS_S3_PLATFORM=${AIRS_S3_PLATFORM}
+      - AIRS_S3_ASSET_HTTP_ENDPOINT_URL=${AIRS_S3_ASSET_HTTP_ENDPOINT_URL}
+      - AIRS_S3_ENDPOINT_URL=${AIRS_S3_ENDPOINT_URL}
+      - AIRS_MAPPING_URL=${AIRS_MAPPING_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json}
+      - AIRS_COLLECTION_URL=${AIRS_COLLECTION_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/collection.json}
+      - AIRS_LOGGER_LEVEL=${AIRS_LOGGER_LEVEL:-INFO}
+    ports:
+      - "8000:8000"
+    healthcheck:
+      test: "curl -f http://localhost:8000/arlas/airs/healthcheck"
+      interval: 5s
+      timeout: 3s
+      retries: 30
 
   aproc-processes:
     networks:
@@ -125,53 +185,26 @@ services:
       - ./test/inputs:/inputs:ro
       - ./outbox/:/outbox
     depends_on:
-      - redis
-      - rabbitmq
-      - smtp4dev
+      redis:
+          condition: service_healthy
+      rabbitmq:
+          condition: service_healthy
 
-  airs-server:
-    networks:
-      - aias
-    build:
-      context: '.'
-      dockerfile: Dockerfile-airs
-    image: airs-server:latest
-    container_name: 'airs-server'
-    environment:
-      - AIRS_HOST=0.0.0.0
-      - AIRS_PORT=8000
-      - AIRS_CORS_ORIGINS=${AIRS_CORS_ORIGINS:-*}
-      - AIRS_CORS_METHODS=${AIRS_CORS_METHODS:-*}
-      - AIRS_CORS_HEADERS=${AIRS_CORS_HEADERS:-*}
-      - AIRS_INDEX_ENDPOINT_URL=${AIRS_INDEX_ENDPOINT_URL:-http://localhost:9200}
-      - AIRS_INDEX_COLLECTION_PREFIX=${AIRS_INDEX_COLLECTION_PREFIX:-airs}
-      - AIRS_INDEX_LOGIN=${AIRS_INDEX_LOGIN:-None}
-      - AIRS_INDEX_PWD=${AIRS_INDEX_PWD:-None}
-      - AIRS_S3_BUCKET=${AIRS_S3_BUCKET:-airstest}
-      - AIRS_S3_ACCESS_KEY_ID=${AIRS_S3_ACCESS_KEY_ID}
-      - AIRS_S3_SECRET_ACCESS_KEY=${AIRS_S3_SECRET_ACCESS_KEY}
-      - AIRS_S3_REGION=${AIRS_S3_REGION:-None}
-      - AIRS_S3_TIER=${AIRS_S3_TIER:-Standard}
-      - AIRS_S3_PLATFORM=${AIRS_S3_PLATFORM}
-      - AIRS_S3_ASSET_HTTP_ENDPOINT_URL=${AIRS_S3_ASSET_HTTP_ENDPOINT_URL}
-      - AIRS_S3_ENDPOINT_URL=${AIRS_S3_ENDPOINT_URL}
-      - AIRS_MAPPING_URL=${AIRS_MAPPING_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json}
-      - AIRS_COLLECTION_URL=${AIRS_COLLECTION_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/collection.json}
-      - AIRS_LOGGER_LEVEL=${AIRS_LOGGER_LEVEL:-INFO}
-    ports:
-      - "8000:8000"
-    depends_on:
-      - elasticsearch
-      - minio
-
-  aproc-server:
+  aproc-service:
     networks:
       - aias
     build:
       context: '.'
       dockerfile: Dockerfile-aproc-service
     image: aproc-service:latest
     container_name: 'aproc-service'
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      redis:
+          condition: service_healthy
+      rabbitmq:
+          condition: service_healthy
     environment:
       - ARLAS_URL_SEARCH=$ARLAS_URL_SEARCH
       - APROC_HOST=0.0.0.0
@@ -214,9 +247,11 @@ services:
       - ./test/inputs:/inputs:ro
     ports:
       - "8001:8001"
-    depends_on:
-      - redis
-      - rabbitmq
+    healthcheck:
+      test: "curl -f http://localhost:8001/arlas/aproc/healthcheck"
+      interval: 5s
+      timeout: 3s
+      retries: 30
 
   aproc-ingest:
     networks:
@@ -226,8 +261,11 @@ services:
       dockerfile: Dockerfile-aproc-ingest
     image: aproc-ingest:latest
     container_name: 'aproc-ingest'
+    depends_on:
+      aproc-service:
+        condition: service_healthy
     environment:
-      - APROC_ENDPOINT=http://aproc-server:8001/arlas/aproc
+      - APROC_ENDPOINT=http://aproc-service:8001/arlas/aproc
       - DRIVERS_CONFIGURATION_FILE=/app/conf/drivers.yaml
       - INGESTED_FOLDER=/inputs
       - OUTPUT_FILE=/tmp/results.csv

diff --git a/extensions/aproc/proc/download/download_process.py b/extensions/aproc/proc/download/download_process.py
@@ -141,6 +141,7 @@ def __get_download_location__(item: Item, send_to: str, format: str) -> (str, st
         if send_to is None: send_to = "anonymous"
         target_directory = os.path.join(Configuration.settings.outbox_directory, send_to.split("@")[0].replace(".","_").replace("-","_"), item.id)
         if not os.path.exists(target_directory):
+            LOGGER.info("create {}".format(target_directory))
             os.makedirs(target_directory)
         file_name = os.path.basename(item.id.replace("-", "_").replace(" ", "_").replace("/", "_").replace("\\", "_").replace("@", "_"))+"."+format
         if os.path.exists(file_name):

diff --git a/extensions/aproc/proc/download/notifications.py b/extensions/aproc/proc/download/notifications.py
@@ -24,8 +24,11 @@ def init():
             LOGGER.info("Index {} does not exists. Attempt to create it with mapping from {} and {}".format(Configuration.settings.index_for_download.index_name, Configuration.settings.arlaseo_mapping_url, Configuration.settings.download_mapping_url))
             mapping = Notifications.__fetch_mapping__(Configuration.settings.arlaseo_mapping_url)
             mapping["properties"]["properties"]["properties"].update(Notifications.__fetch_mapping__(Configuration.settings.download_mapping_url)["properties"]["properties"]["properties"])
-            Notifications.__getES().indices.create(index=Configuration.settings.index_for_download.index_name, mappings=mapping)
-            LOGGER.info("Mapping {} updated.".format(Configuration.settings.index_for_download.index_name))
+            try:
+                Notifications.__getES().indices.create(index=Configuration.settings.index_for_download.index_name, mappings=mapping)
+                LOGGER.info("Mapping {} updated.".format(Configuration.settings.index_for_download.index_name))
+            except Exception as e:
+                LOGGER.error("Can not create {} ({})".format(Configuration.settings.index_for_download.index_name, e))
         else:
             LOGGER.debug("Index {} found.".format(Configuration.settings.index_for_download.index_name))
 

diff --git a/test/agate_tests.py b/test/agate_tests.py
@@ -1,6 +1,7 @@
 import json
 import os
 import unittest
+from time import sleep
 
 import requests
 from utils import (AGATE_ENDPOINT, AIRS_URL, ARLAS_COLLECTION, ARLAS_URL,
@@ -55,6 +56,7 @@ def __add_item__(self) -> Item:
         print("item created")
         r = requests.get(url=os.path.join(AIRS_URL, "collections", COLLECTION, "items", ID))
         self.assertTrue(r.ok, msg=r.content)
+        sleep(3)
         return mapper.item_from_json(r.content)
 
 

diff --git a/test/aproc_ingest_tests.py b/test/aproc_ingest_tests.py
@@ -41,10 +41,6 @@ def ingest_directory(self, url, collection, catalog):
             status: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content))
         self.assertEqual(status.status, StatusCode.successful)
 
-    def test_async_ingest_theia(self):
-        url = "https://catalogue.theia-land.fr/arlas/explore/theia/_search?f=metadata.core.identity.identifier%3Aeq%3ASENTINEL2A_20230604-105902-526_L2A_T31TCJ_D&righthand=false&pretty=false&flat=false&&&size=1&max-age-cache=120"
-        self.ingest(url, COLLECTION, "theia")
-
     def test_async_ingest_dimap(self):
         url = "/inputs/DIMAP/PROD_SPOT6_001/VOL_SPOT6_001_A/IMG_SPOT6_MS_001_A/"
         self.ingest(url, COLLECTION, "spot6")
@@ -67,8 +63,8 @@ def test_jobs(self):
         r = requests.get("/".join([APROC_ENDPOINT, "jobs"]))
         self.assertTrue(r.ok, str(r.status_code) + ": " + str(r.content))
 
-    def __ingest_theia(self) -> StatusInfo:
-        url = "https://catalogue.theia-land.fr/arlas/explore/theia/_search?f=metadata.core.identity.identifier%3Aeq%3ASENTINEL2A_20230604-105902-526_L2A_T31TCJ_D&righthand=false&pretty=false&flat=false&&&size=1&max-age-cache=120"
+    def __ingest_dimap(self) -> StatusInfo:
+        url = "/inputs/DIMAP/PROD_SPOT6_001/VOL_SPOT6_001_A/IMG_SPOT6_MS_001_A/"
         collection = COLLECTION
         catalog = "theia"
         inputs = InputIngestProcess(url=url, collection=collection, catalog=catalog, annotations="")
@@ -78,21 +74,21 @@ def __ingest_theia(self) -> StatusInfo:
         return StatusInfo(**json.loads(r.content))
 
     def test_job_by_id(self):
-        status: StatusInfo = self.__ingest_theia()
+        status: StatusInfo = self.__ingest_dimap()
         status2: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content))
         self.assertEqual(status.jobID, status2.jobID)
         self.assertEqual(status2.processID, "ingest")
 
     def test_job_result(self):
-        status: StatusInfo = self.__ingest_theia()
+        status: StatusInfo = self.__ingest_dimap()
         while status.status not in [StatusCode.failed, StatusCode.dismissed, StatusCode.successful]:
             sleep(1)
             status: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content))
         result = json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID, "results"])).content)
-        self.assertEqual(result["item_location"], "http://airs-server:8000/arlas/airs/collections/"+COLLECTION+"/items/SENTINEL2A_20230604-105902-526_L2A_T31TCJ_D", result["item_location"])
+        self.assertEqual(result["item_location"], "http://airs-server:8000/arlas/airs/collections/"+COLLECTION+"/items/148ddaaa431bdd2ff06b823df1e3725d462f668bd95188603bfff443ff055c71", result["item_location"])
 
     def test_get_jobs_by_resource_id(self):
-        status: StatusInfo = self.__ingest_theia()
+        status: StatusInfo = self.__ingest_dimap()
         resource_status: list = json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs/resources", status.resourceID])).content)
         self.assertGreaterEqual(len(resource_status), 1)
         self.assertEqual(resource_status[0]["resourceID"], status.resourceID)

diff --git a/test/requirements.txt b/test/requirements.txt
@@ -8,3 +8,8 @@ envyaml==1.10.211231
 requests==2.31.0
 ecs-logging==2.1.0
 uvicorn==0.23.1
+pygeohash==1.2.0
+pytz==2023.3
+celery==5.3.1
+celery[redis]==5.3.1
+jsonref==1.1.0
diff --git a/test/start_stack.sh b/test/start_stack.sh
@@ -1,5 +1,6 @@
+#!/usr/bin/env sh
 # Set env variable
-source ./test/env.sh
+. ./test/env.sh
 # Copy heavy data for test from gcp bucket
 if [ -d "${ROOT_DIRECTORY}/DIMAP" ]; then
     echo "${ROOT_DIRECTORY}/DIMAP exists, files are not downloaded."
@@ -11,30 +12,6 @@ rm -rf ./outbox
 mkdir outbox
 
 # Start  minio
-docker compose -f docker-compose.yaml up  -d minio
-#Waiting for minio service up and running
-code=""
-code_OK="OK"
- while [[ "$code" != *$code_OK* ]];do
-    code="$(curl -IL --silent http://localhost:9000/minio/health/live | grep "^HTTP\/")"
-    eval "sleep 5"
-done
-
-# Start  create buckets, elastic rabbitmq redis airs, aproc
-docker compose -f docker-compose.yaml up --build -d createbuckets elasticsearch rabbitmq smtp4dev redis aproc-processes airs-server aproc-server fam-service
-# Waiting for elastic ready
-code=""
-code_OK="OK"
-while [[ "$code" != *$code_OK* ]];do
-    code="$(curl -IL --silent http://localhost:9200 | grep "^HTTP\/")"
-    eval "sleep 5"
-done
-docker compose -f docker-compose-tests.yaml up --build -d
-# Waiting for arlas ready
-code=""
-code_OK="OK"
-while [[ "$code" != *$code_OK* ]];do
-    code="$(curl -IL --silent http://localhost:9999 | grep "^HTTP\/")"
-    eval "sleep 5"
-done
+docker compose -f docker-compose.yaml up -d --remove-orphans --build --wait || true
 
+docker compose -f docker-compose.yaml -f docker-compose-tests.yaml up --build --wait || true
diff --git a/test/stop_stack.sh b/test/stop_stack.sh
@@ -1,4 +1,3 @@
 source ./test/env.sh
-docker compose -f docker-compose-tests.yaml down
-docker compose -f docker-compose.yaml down
+docker compose -f docker-compose.yaml -f docker-compose-tests.yaml down
 rm -rf ./outbox
diff --git a/test/tests.sh b/test/tests.sh
@@ -1,6 +1,10 @@
+#!/usr/bin/env sh
+set -o errexit
+
 ./test/start_stack.sh
+
 # Set env variable
-source ./test/env.sh
+. ./test/env.sh
 python3 test/airs_tests.py
 python3 test/aproc_ingest_tests.py
 python3 test/aproc_download_tests.py