From 3256d8ae7a03e9a4b05c07048e5860968c4912b0 Mon Sep 17 00:00:00 2001 From: Sylvain Gaudan <35499583+sylvaingaudan@users.noreply.github.com> Date: Tue, 30 Apr 2024 19:18:49 +0200 Subject: [PATCH] Bug/ci: must fail on error (#139) * add pygeohash * add pytz * fail ci on error * add celery * remove pipefail * add jsonref * start * little sleep for agate * sh to bash * [[ to [ * test * refactor dc * exit on error --- docker-compose-tests.yaml | 19 ++- docker-compose.yaml | 126 ++++++++++++------ .../aproc/proc/download/download_process.py | 1 + .../aproc/proc/download/notifications.py | 7 +- test/agate_tests.py | 2 + test/aproc_ingest_tests.py | 16 +-- test/requirements.txt | 5 + test/start_stack.sh | 31 +---- test/stop_stack.sh | 3 +- test/tests.sh | 6 +- 10 files changed, 124 insertions(+), 92 deletions(-) diff --git a/docker-compose-tests.yaml b/docker-compose-tests.yaml index 16d2d1e5..9c4d05fc 100644 --- a/docker-compose-tests.yaml +++ b/docker-compose-tests.yaml @@ -16,9 +16,12 @@ services: - ServerOptions__HostName=smtp4dev arlas-server: - image: gisaia/arlas-server:${ARLAS_SERVER_VERSION:-23.0.7} + image: gisaia/arlas-server:${ARLAS_SERVER_VERSION:-24.1.2} platform: linux/amd64 container_name: arlas-server + depends_on: + elasticsearch: + condition: service_healthy networks: - aias ports: @@ -35,7 +38,12 @@ services: - ARLAS_ELASTIC_INDEX=.arlas - ARLAS_APP_PATH=/ - ARLAS_BASE_URI=http://localhost:9999/server/ - + healthcheck: + test: ["CMD","java","HttpHealthcheck.java","http://localhost:9999/admin/healthcheck"] + interval: 10s + timeout: 10s + retries: 6 + agate: networks: - aias @@ -44,6 +52,9 @@ services: dockerfile: Dockerfile-agate image: agate:latest container_name: 'agate' + depends_on: + arlas-server: + condition: service_healthy environment: - AGATE_CORS_ORIGINS=${AGATE_CORS_ORIGINS:-*} - AGATE_CORS_METHODS=${AGATE_CORS_METHODS:-*} @@ -59,7 +70,3 @@ services: # - ASSET_MINIO_PUBLIC_PATTERN="(/collections/)(?P[^/]+)/items/(?P[^/]+)/assets/thumbnail" ports: - $AGATE_PORT:$AGATE_PORT - depends_on: - - arlas-server - - diff --git a/docker-compose.yaml b/docker-compose.yaml index b8ac6f6f..62d85522 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,11 +14,17 @@ services: - "MINIO_ACCESS_KEY=airs" - "MINIO_SECRET_KEY=airssecret" command: server /export --console-address ":9001" + healthcheck: + test: "mc alias set myminio http://localhost:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; mc ping -x myminio" + interval: 5s + timeout: 10s + retries: 3 createbuckets: image: minio/mc depends_on: - - minio + minio: + condition: service_healthy environment: - BUCKET_NAME=${BUCKET_NAME:-airstest} - LOGIN=${LOGIN:-airs} @@ -47,6 +53,11 @@ services: - xpack.security.transport.ssl.enabled=false ports: - 9200:9200 + healthcheck: + test: "curl -s -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'" + interval: 10s + timeout: 10s + retries: 24 rabbitmq: image: rabbitmq:${RABBITMQ_VERSION:-3-management} @@ -56,6 +67,11 @@ services: ports: - 15672:15672 - 5672:5672 + healthcheck: + test: rabbitmq-diagnostics check_port_connectivity + interval: 30s + timeout: 30s + retries: 10 smtp4dev: image: rnwood/smtp4dev:linux-${PLATFORM}-3.2.0-ci20221023104 @@ -76,6 +92,50 @@ services: container_name: redis ports: - 6379:6379 + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + + airs-server: + networks: + - aias + build: + context: '.' + dockerfile: Dockerfile-airs + image: airs-server:latest + container_name: 'airs-server' + depends_on: + elasticsearch: + condition: service_healthy + minio: + condition: service_healthy + environment: + - AIRS_HOST=0.0.0.0 + - AIRS_PORT=8000 + - AIRS_CORS_ORIGINS=${AIRS_CORS_ORIGINS:-*} + - AIRS_CORS_METHODS=${AIRS_CORS_METHODS:-*} + - AIRS_CORS_HEADERS=${AIRS_CORS_HEADERS:-*} + - AIRS_INDEX_ENDPOINT_URL=${AIRS_INDEX_ENDPOINT_URL:-http://localhost:9200} + - AIRS_INDEX_COLLECTION_PREFIX=${AIRS_INDEX_COLLECTION_PREFIX:-airs} + - AIRS_INDEX_LOGIN=${AIRS_INDEX_LOGIN:-None} + - AIRS_INDEX_PWD=${AIRS_INDEX_PWD:-None} + - AIRS_S3_BUCKET=${AIRS_S3_BUCKET:-airstest} + - AIRS_S3_ACCESS_KEY_ID=${AIRS_S3_ACCESS_KEY_ID} + - AIRS_S3_SECRET_ACCESS_KEY=${AIRS_S3_SECRET_ACCESS_KEY} + - AIRS_S3_REGION=${AIRS_S3_REGION:-None} + - AIRS_S3_TIER=${AIRS_S3_TIER:-Standard} + - AIRS_S3_PLATFORM=${AIRS_S3_PLATFORM} + - AIRS_S3_ASSET_HTTP_ENDPOINT_URL=${AIRS_S3_ASSET_HTTP_ENDPOINT_URL} + - AIRS_S3_ENDPOINT_URL=${AIRS_S3_ENDPOINT_URL} + - AIRS_MAPPING_URL=${AIRS_MAPPING_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json} + - AIRS_COLLECTION_URL=${AIRS_COLLECTION_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/collection.json} + - AIRS_LOGGER_LEVEL=${AIRS_LOGGER_LEVEL:-INFO} + ports: + - "8000:8000" + healthcheck: + test: "curl -f http://localhost:8000/arlas/airs/healthcheck" + interval: 5s + timeout: 3s + retries: 30 aproc-processes: networks: @@ -125,46 +185,12 @@ services: - ./test/inputs:/inputs:ro - ./outbox/:/outbox depends_on: - - redis - - rabbitmq - - smtp4dev + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy - airs-server: - networks: - - aias - build: - context: '.' - dockerfile: Dockerfile-airs - image: airs-server:latest - container_name: 'airs-server' - environment: - - AIRS_HOST=0.0.0.0 - - AIRS_PORT=8000 - - AIRS_CORS_ORIGINS=${AIRS_CORS_ORIGINS:-*} - - AIRS_CORS_METHODS=${AIRS_CORS_METHODS:-*} - - AIRS_CORS_HEADERS=${AIRS_CORS_HEADERS:-*} - - AIRS_INDEX_ENDPOINT_URL=${AIRS_INDEX_ENDPOINT_URL:-http://localhost:9200} - - AIRS_INDEX_COLLECTION_PREFIX=${AIRS_INDEX_COLLECTION_PREFIX:-airs} - - AIRS_INDEX_LOGIN=${AIRS_INDEX_LOGIN:-None} - - AIRS_INDEX_PWD=${AIRS_INDEX_PWD:-None} - - AIRS_S3_BUCKET=${AIRS_S3_BUCKET:-airstest} - - AIRS_S3_ACCESS_KEY_ID=${AIRS_S3_ACCESS_KEY_ID} - - AIRS_S3_SECRET_ACCESS_KEY=${AIRS_S3_SECRET_ACCESS_KEY} - - AIRS_S3_REGION=${AIRS_S3_REGION:-None} - - AIRS_S3_TIER=${AIRS_S3_TIER:-Standard} - - AIRS_S3_PLATFORM=${AIRS_S3_PLATFORM} - - AIRS_S3_ASSET_HTTP_ENDPOINT_URL=${AIRS_S3_ASSET_HTTP_ENDPOINT_URL} - - AIRS_S3_ENDPOINT_URL=${AIRS_S3_ENDPOINT_URL} - - AIRS_MAPPING_URL=${AIRS_MAPPING_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json} - - AIRS_COLLECTION_URL=${AIRS_COLLECTION_URL:-https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/collection.json} - - AIRS_LOGGER_LEVEL=${AIRS_LOGGER_LEVEL:-INFO} - ports: - - "8000:8000" - depends_on: - - elasticsearch - - minio - - aproc-server: + aproc-service: networks: - aias build: @@ -172,6 +198,13 @@ services: dockerfile: Dockerfile-aproc-service image: aproc-service:latest container_name: 'aproc-service' + depends_on: + elasticsearch: + condition: service_healthy + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy environment: - ARLAS_URL_SEARCH=$ARLAS_URL_SEARCH - APROC_HOST=0.0.0.0 @@ -214,9 +247,11 @@ services: - ./test/inputs:/inputs:ro ports: - "8001:8001" - depends_on: - - redis - - rabbitmq + healthcheck: + test: "curl -f http://localhost:8001/arlas/aproc/healthcheck" + interval: 5s + timeout: 3s + retries: 30 aproc-ingest: networks: @@ -226,8 +261,11 @@ services: dockerfile: Dockerfile-aproc-ingest image: aproc-ingest:latest container_name: 'aproc-ingest' + depends_on: + aproc-service: + condition: service_healthy environment: - - APROC_ENDPOINT=http://aproc-server:8001/arlas/aproc + - APROC_ENDPOINT=http://aproc-service:8001/arlas/aproc - DRIVERS_CONFIGURATION_FILE=/app/conf/drivers.yaml - INGESTED_FOLDER=/inputs - OUTPUT_FILE=/tmp/results.csv diff --git a/extensions/aproc/proc/download/download_process.py b/extensions/aproc/proc/download/download_process.py index c1c545a2..7bf05d04 100644 --- a/extensions/aproc/proc/download/download_process.py +++ b/extensions/aproc/proc/download/download_process.py @@ -141,6 +141,7 @@ def __get_download_location__(item: Item, send_to: str, format: str) -> (str, st if send_to is None: send_to = "anonymous" target_directory = os.path.join(Configuration.settings.outbox_directory, send_to.split("@")[0].replace(".","_").replace("-","_"), item.id) if not os.path.exists(target_directory): + LOGGER.info("create {}".format(target_directory)) os.makedirs(target_directory) file_name = os.path.basename(item.id.replace("-", "_").replace(" ", "_").replace("/", "_").replace("\\", "_").replace("@", "_"))+"."+format if os.path.exists(file_name): diff --git a/extensions/aproc/proc/download/notifications.py b/extensions/aproc/proc/download/notifications.py index bb5e3176..bdd175bc 100644 --- a/extensions/aproc/proc/download/notifications.py +++ b/extensions/aproc/proc/download/notifications.py @@ -24,8 +24,11 @@ def init(): LOGGER.info("Index {} does not exists. Attempt to create it with mapping from {} and {}".format(Configuration.settings.index_for_download.index_name, Configuration.settings.arlaseo_mapping_url, Configuration.settings.download_mapping_url)) mapping = Notifications.__fetch_mapping__(Configuration.settings.arlaseo_mapping_url) mapping["properties"]["properties"]["properties"].update(Notifications.__fetch_mapping__(Configuration.settings.download_mapping_url)["properties"]["properties"]["properties"]) - Notifications.__getES().indices.create(index=Configuration.settings.index_for_download.index_name, mappings=mapping) - LOGGER.info("Mapping {} updated.".format(Configuration.settings.index_for_download.index_name)) + try: + Notifications.__getES().indices.create(index=Configuration.settings.index_for_download.index_name, mappings=mapping) + LOGGER.info("Mapping {} updated.".format(Configuration.settings.index_for_download.index_name)) + except Exception as e: + LOGGER.error("Can not create {} ({})".format(Configuration.settings.index_for_download.index_name, e)) else: LOGGER.debug("Index {} found.".format(Configuration.settings.index_for_download.index_name)) diff --git a/test/agate_tests.py b/test/agate_tests.py index 4ce2cf36..430c9cf1 100644 --- a/test/agate_tests.py +++ b/test/agate_tests.py @@ -1,6 +1,7 @@ import json import os import unittest +from time import sleep import requests from utils import (AGATE_ENDPOINT, AIRS_URL, ARLAS_COLLECTION, ARLAS_URL, @@ -55,6 +56,7 @@ def __add_item__(self) -> Item: print("item created") r = requests.get(url=os.path.join(AIRS_URL, "collections", COLLECTION, "items", ID)) self.assertTrue(r.ok, msg=r.content) + sleep(3) return mapper.item_from_json(r.content) diff --git a/test/aproc_ingest_tests.py b/test/aproc_ingest_tests.py index 65679c5a..178dad7c 100644 --- a/test/aproc_ingest_tests.py +++ b/test/aproc_ingest_tests.py @@ -41,10 +41,6 @@ def ingest_directory(self, url, collection, catalog): status: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content)) self.assertEqual(status.status, StatusCode.successful) - def test_async_ingest_theia(self): - url = "https://catalogue.theia-land.fr/arlas/explore/theia/_search?f=metadata.core.identity.identifier%3Aeq%3ASENTINEL2A_20230604-105902-526_L2A_T31TCJ_D&righthand=false&pretty=false&flat=false&&&size=1&max-age-cache=120" - self.ingest(url, COLLECTION, "theia") - def test_async_ingest_dimap(self): url = "/inputs/DIMAP/PROD_SPOT6_001/VOL_SPOT6_001_A/IMG_SPOT6_MS_001_A/" self.ingest(url, COLLECTION, "spot6") @@ -67,8 +63,8 @@ def test_jobs(self): r = requests.get("/".join([APROC_ENDPOINT, "jobs"])) self.assertTrue(r.ok, str(r.status_code) + ": " + str(r.content)) - def __ingest_theia(self) -> StatusInfo: - url = "https://catalogue.theia-land.fr/arlas/explore/theia/_search?f=metadata.core.identity.identifier%3Aeq%3ASENTINEL2A_20230604-105902-526_L2A_T31TCJ_D&righthand=false&pretty=false&flat=false&&&size=1&max-age-cache=120" + def __ingest_dimap(self) -> StatusInfo: + url = "/inputs/DIMAP/PROD_SPOT6_001/VOL_SPOT6_001_A/IMG_SPOT6_MS_001_A/" collection = COLLECTION catalog = "theia" inputs = InputIngestProcess(url=url, collection=collection, catalog=catalog, annotations="") @@ -78,21 +74,21 @@ def __ingest_theia(self) -> StatusInfo: return StatusInfo(**json.loads(r.content)) def test_job_by_id(self): - status: StatusInfo = self.__ingest_theia() + status: StatusInfo = self.__ingest_dimap() status2: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content)) self.assertEqual(status.jobID, status2.jobID) self.assertEqual(status2.processID, "ingest") def test_job_result(self): - status: StatusInfo = self.__ingest_theia() + status: StatusInfo = self.__ingest_dimap() while status.status not in [StatusCode.failed, StatusCode.dismissed, StatusCode.successful]: sleep(1) status: StatusInfo = StatusInfo(**json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID])).content)) result = json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs", status.jobID, "results"])).content) - self.assertEqual(result["item_location"], "http://airs-server:8000/arlas/airs/collections/"+COLLECTION+"/items/SENTINEL2A_20230604-105902-526_L2A_T31TCJ_D", result["item_location"]) + self.assertEqual(result["item_location"], "http://airs-server:8000/arlas/airs/collections/"+COLLECTION+"/items/148ddaaa431bdd2ff06b823df1e3725d462f668bd95188603bfff443ff055c71", result["item_location"]) def test_get_jobs_by_resource_id(self): - status: StatusInfo = self.__ingest_theia() + status: StatusInfo = self.__ingest_dimap() resource_status: list = json.loads(requests.get("/".join([APROC_ENDPOINT, "jobs/resources", status.resourceID])).content) self.assertGreaterEqual(len(resource_status), 1) self.assertEqual(resource_status[0]["resourceID"], status.resourceID) diff --git a/test/requirements.txt b/test/requirements.txt index e5891350..11e400ca 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -8,3 +8,8 @@ envyaml==1.10.211231 requests==2.31.0 ecs-logging==2.1.0 uvicorn==0.23.1 +pygeohash==1.2.0 +pytz==2023.3 +celery==5.3.1 +celery[redis]==5.3.1 +jsonref==1.1.0 \ No newline at end of file diff --git a/test/start_stack.sh b/test/start_stack.sh index 62d2abf8..8d33a96d 100755 --- a/test/start_stack.sh +++ b/test/start_stack.sh @@ -1,5 +1,6 @@ +#!/usr/bin/env sh # Set env variable -source ./test/env.sh +. ./test/env.sh # Copy heavy data for test from gcp bucket if [ -d "${ROOT_DIRECTORY}/DIMAP" ]; then echo "${ROOT_DIRECTORY}/DIMAP exists, files are not downloaded." @@ -11,30 +12,6 @@ rm -rf ./outbox mkdir outbox # Start minio -docker compose -f docker-compose.yaml up -d minio -#Waiting for minio service up and running -code="" -code_OK="OK" - while [[ "$code" != *$code_OK* ]];do - code="$(curl -IL --silent http://localhost:9000/minio/health/live | grep "^HTTP\/")" - eval "sleep 5" -done - -# Start create buckets, elastic rabbitmq redis airs, aproc -docker compose -f docker-compose.yaml up --build -d createbuckets elasticsearch rabbitmq smtp4dev redis aproc-processes airs-server aproc-server fam-service -# Waiting for elastic ready -code="" -code_OK="OK" -while [[ "$code" != *$code_OK* ]];do - code="$(curl -IL --silent http://localhost:9200 | grep "^HTTP\/")" - eval "sleep 5" -done -docker compose -f docker-compose-tests.yaml up --build -d -# Waiting for arlas ready -code="" -code_OK="OK" -while [[ "$code" != *$code_OK* ]];do - code="$(curl -IL --silent http://localhost:9999 | grep "^HTTP\/")" - eval "sleep 5" -done +docker compose -f docker-compose.yaml up -d --remove-orphans --build --wait || true +docker compose -f docker-compose.yaml -f docker-compose-tests.yaml up --build --wait || true diff --git a/test/stop_stack.sh b/test/stop_stack.sh index a141d7ce..6d12777f 100755 --- a/test/stop_stack.sh +++ b/test/stop_stack.sh @@ -1,4 +1,3 @@ source ./test/env.sh -docker compose -f docker-compose-tests.yaml down -docker compose -f docker-compose.yaml down +docker compose -f docker-compose.yaml -f docker-compose-tests.yaml down rm -rf ./outbox diff --git a/test/tests.sh b/test/tests.sh index 9c3417e6..a9f05dcd 100755 --- a/test/tests.sh +++ b/test/tests.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env sh +set -o errexit + ./test/start_stack.sh + # Set env variable -source ./test/env.sh +. ./test/env.sh python3 test/airs_tests.py python3 test/aproc_ingest_tests.py python3 test/aproc_download_tests.py