diff --git a/.gitignore b/.gitignore index 6105c23..93f210f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ conf/apisix/apisix.generated.yaml conf/apisix/apisix.generated.tmp.yaml sample/dashboard.generated.json tmp/ +docker-compose.env diff --git a/conf/aias.env b/conf/aias.env index 22f503d..15682ab 100644 --- a/conf/aias.env +++ b/conf/aias.env @@ -6,7 +6,7 @@ AIRS_S3_ACCESS_KEY_ID=airs AIRS_S3_ASSET_HTTP_ENDPOINT_URL=http://minio:9000/{}/{} AIRS_S3_BUCKET=airs AIRS_S3_SECRET_ACCESS_KEY=airssecret -AIRS_STORAGE_DIRECTORY= +AIRS_STORAGE_DIRECTORY=arlas-data-minio AIRS_S3_ENDPOINT_URL=http://minio:9000 APROC_DOWNLOAD_DIR=/tmp/outbox @@ -48,6 +48,23 @@ APROC_INDEX_NAME=aproc_downloads APROC_RESOURCE_ID_HASH_STARTS_AT=1 ARLAS_URL_SEARCH="http://arlas-server:9999/arlas/explore/{collection}/_search?f=id:eq:{item}" + +ARLAS_FAM_LINKS=' + [ + { + "name":"Hub", + "url":"/hub/", + "icon":"hub", + "check_url": "/arlas_persistence_server/healthcheck" + }, + { + "name":"IAM", + "url":"/iam/", + "icon":"manage_accounts", + "check_url": "/arlas_iam_server/healthcheck" + } + ]' + #AIRS_HOST= #AIRS_INDEX_COLLECTION_PREFIX= #AIRS_MAPPING_URL= diff --git a/conf/aias/drivers.yaml b/conf/aias/drivers.yaml index b999472..60cf628 100644 --- a/conf/aias/drivers.yaml +++ b/conf/aias/drivers.yaml @@ -56,30 +56,18 @@ drivers: tmp_directory: $TMP_FOLDER|/tmp priority: 8 - - - name: theia - class_name: extensions.aproc.proc.ingest.drivers.impl.theia - assets_dir: /tmp/aproc/theia + name: tiff + class_name: extensions.aproc.proc.ingest.drivers.impl.tiff + assets_dir: /tmp/aproc/tiff configuration: - token_url: https://theia.cnes.fr/atdistrib/services/authenticate/ - login: $THEIA_LOGIN - pwd: $THEIA_PWD - manage_data: false - priority: 7 - - # - - # name: tiff - # class_name: extensions.aproc.proc.ingest.drivers.impl.tiff - # assets_dir: /tmp/aproc/tiff - # configuration: - # tmp_directory: $TMP_FOLDER|/tmp - # priority: 9 + tmp_directory: $TMP_FOLDER|/tmp + priority: 9 - # - - # name: jpeg2000 - # class_name: extensions.aproc.proc.ingest.drivers.impl.jpeg2000 - # assets_dir: /tmp/aproc/jpeg2000 - # configuration: - # tmp_directory: $TMP_FOLDER|/tmp - # priority: 10 + - + name: jpeg2000 + class_name: extensions.aproc.proc.ingest.drivers.impl.jpeg2000 + assets_dir: /tmp/aproc/jpeg2000 + configuration: + tmp_directory: $TMP_FOLDER|/tmp + priority: 10 diff --git a/conf/apisix/apisix.yaml b/conf/apisix/apisix.yaml index 4f45231..5b17295 100644 --- a/conf/apisix/apisix.yaml +++ b/conf/apisix/apisix.yaml @@ -337,6 +337,143 @@ routes: Strict-Transport-Security: max-age=63072000; includeSubDomains; preload # Fix CWE Id: 693 X-Content-Type-Options: nosniff + - + uri: /fam-wui + plugins: + redirect: + uri: /fam-wui/ + response-rewrite: + headers: + set: + # Fix CWE Id: 693 + Content-Security-Policy: "default-src https: data: blob: 'self'; connect-src 'self' data: blob:; media-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; font-src 'self'; img-src 'self' data: blob:; frame-ancestors 'none'; form-action 'self'; object-src 'none'; manifest-src 'self'; frame-src 'self'" + # Fix CWE Id: 200 + Server: "ARLAS services" + # Fix CWE Id: 319 // Value recommended by https://hstspreload.org/ + Strict-Transport-Security: max-age=63072000; includeSubDomains; preload + # Fix CWE Id: 693 + X-Content-Type-Options: nosniff + - + uri: /fam-wui/* + upstream: + nodes: + "arlas-fam-wui:80": 1 + plugins: + response-rewrite: + headers: + set: + # Fix CWE Id: 693 + Content-Security-Policy: "default-src https: data: blob: 'self'; connect-src 'self' data: blob:; media-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; font-src 'self'; img-src 'self' data: blob:; frame-ancestors 'none'; form-action 'self'; object-src 'none'; manifest-src 'self'; frame-src 'self'" + # Fix CWE Id: 1021 + X-Frame-Options: "DENY" + # Fix CWE Id: 693 // The () means those resources are not allowed. + Permissions-Policy: accelerometer=(), camera=(), microphone=(), geolocation=(), usb=() + # Fix CWE Id: 200 + Server: "ARLAS services" + # Fix CWE Id: 319 // Value recommended by https://hstspreload.org/ + Strict-Transport-Security: max-age=63072000; includeSubDomains; preload + # Fix CWE Id: 693 + X-Content-Type-Options: nosniff + proxy-rewrite: + regex_uri: ["/fam-wui/(.*)", "/$1"] + - + uri: /agate/healthcheck + methods: ["GET"] + upstream: + nodes: + "agate:8004": 1 + - + uri: /airs-storage/* + methods: ["GET"] + upstream: + nodes: + "minio:9000": 1 + plugins: + forward-auth: + uri: http://agate:8004/agate/authorization + request_headers: ["Authorization", "arlas-org-filter"] + - + uri: /fam/* + upstream: + nodes: + "fam-service:8005": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + - + uri: /fam/files + upstream: + nodes: + "fam-service:8005": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + # proxy-rewrite: + # headers: + # set: + # # Fix CWE Id: 1021 + # Access-Control-Allow-Origin: "https://${ARLAS_HOST}" + # response-rewrite: + # headers: + # set: + # # Fix CWE Id: 693 + # Content-Security-Policy: "default-src https: data: blob: 'self'; connect-src 'self' data: blob:; media-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; font-src 'self'; img-src 'self' data: blob:; frame-ancestors 'none'; form-action 'self'; object-src 'none'; manifest-src 'self'; frame-src 'self'" + # # Fix CWE Id: 200 + # Server: "ARLAS services" + # # Fix CWE Id: 319 // Value recommended by https://hstspreload.org/ + # Strict-Transport-Security: max-age=63072000; includeSubDomains; preload + # # Fix CWE Id: 693 + # X-Content-Type-Options: nosniff + - + uri: /airs/healthcheck + methods: ["GET"] + upstream: + nodes: + "airs-server:8000": 1 + - + uri: /airs/collections/* + upstream: + nodes: + "airs-server:8000": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + - + uri: /aproc/healthcheck + methods: ["GET"] + upstream: + nodes: + "aproc-service:8001": 1 + - + uri: /aproc/processes/* + upstream: + nodes: + "aproc-service:8001": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + - + uri: /aproc/jobs + upstream: + nodes: + "aproc-service:8001": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + - + uri: /aproc/jobs/* + upstream: + nodes: + "aproc-service:8001": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] ssls: - snis: diff --git a/conf/apisix/apisix_part_aias_services.yaml b/conf/apisix/apisix_part_aias_services.yaml index db4d64e..0b5d2ad 100644 --- a/conf/apisix/apisix_part_aias_services.yaml +++ b/conf/apisix/apisix_part_aias_services.yaml @@ -62,23 +62,31 @@ forward-auth: uri: http://arlas-iam-server:9998/arlas_iam_server/auth request_headers: ["Authorization", "arlas-org-filter"] - proxy-rewrite: - headers: - set: - # Fix CWE Id: 1021 - Access-Control-Allow-Origin: "https://${ARLAS_HOST}" - response-rewrite: - headers: - set: - # Fix CWE Id: 693 - Content-Security-Policy: "default-src https: data: blob: 'self'; connect-src 'self' data: blob:; media-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; font-src 'self'; img-src 'self' data: blob:; frame-ancestors 'none'; form-action 'self'; object-src 'none'; manifest-src 'self'; frame-src 'self'" - # Fix CWE Id: 200 - Server: "ARLAS services" - # Fix CWE Id: 319 // Value recommended by https://hstspreload.org/ - Strict-Transport-Security: max-age=63072000; includeSubDomains; preload - # Fix CWE Id: 693 - X-Content-Type-Options: nosniff - + - + uri: /fam/files + upstream: + nodes: + "fam-service:8005": 1 + plugins: + forward-auth: + uri: http://arlas-iam-server:9998/arlas_iam_server/auth + request_headers: ["Authorization", "arlas-org-filter"] + # proxy-rewrite: + # headers: + # set: + # # Fix CWE Id: 1021 + # Access-Control-Allow-Origin: "https://${ARLAS_HOST}" + # response-rewrite: + # headers: + # set: + # # Fix CWE Id: 693 + # Content-Security-Policy: "default-src https: data: blob: 'self'; connect-src 'self' data: blob:; media-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; font-src 'self'; img-src 'self' data: blob:; frame-ancestors 'none'; form-action 'self'; object-src 'none'; manifest-src 'self'; frame-src 'self'" + # # Fix CWE Id: 200 + # Server: "ARLAS services" + # # Fix CWE Id: 319 // Value recommended by https://hstspreload.org/ + # Strict-Transport-Security: max-age=63072000; includeSubDomains; preload + # # Fix CWE Id: 693 + # X-Content-Type-Options: nosniff - uri: /airs/healthcheck methods: ["GET"] diff --git a/conf/apisix/apisix_with_aias.yaml b/conf/apisix/apisix_with_aias.yaml deleted file mode 100644 index fadb9c5..0000000 --- a/conf/apisix/apisix_with_aias.yaml +++ /dev/null @@ -1,144 +0,0 @@ -routes: - - - uri: /builder - upstream: - nodes: - "arlas-builder:80": 1 - - - uri: /builder/* - upstream: - nodes: - "arlas-builder:80": 1 - plugins: - proxy-rewrite: - regex_uri: ["/builder/(.*)", "/$1"] - - - uri: / - plugins: - redirect: - uri: /hub/ - - - uri: /hub - upstream: - nodes: - "arlas-hub:80": 1 - - - uri: /hub/* - upstream: - nodes: - "arlas-hub:80": 1 - plugins: - proxy-rewrite: - regex_uri: ["/hub/(.*)", "/$1"] - - - uri: /iam - upstream: - nodes: - "arlas-wui-iam:80": 1 - - - uri: /iam/* - upstream: - nodes: - "arlas-wui-iam:80": 1 - - - uri: /wui - upstream: - nodes: - "arlas-wui:80": 1 - - - uri: /wui/* - upstream: - nodes: - "arlas-wui:80": 1 - plugins: - proxy-rewrite: - regex_uri: ["/wui/(.*)", "/$1"] - - - uri: /arlas_permissions_server/* - upstream: - nodes: - "arlas-permissions-server:9996": 1 - - - uri: /persist/* - upstream: - nodes: - "arlas-persistence-server:9997": 1 - - - uri: /arlas/* - upstream: - nodes: - "arlas-server:9999": 1 - - - uri: /agate/healthcheck - methods: ["GET"] - upstream: - nodes: - "agate:8004": 1 - - - uri: /airs-storage/* - methods: ["GET"] - upstream: - nodes: - "minio:9000": 1 - plugins: - forward-auth: - uri: http://agate:8004/agate/authorization - request_headers: ["Authorization", "arlas-org-filter"] - - - uri: /fam-wui - plugins: - redirect: - uri: /fam-wui/ - - - uri: /fam-wui/* - upstream: - nodes: - "arlas-fam-wui:80": 1 - plugins: - proxy-rewrite: - regex_uri: ["/fam-wui/(.*)", "/$1"] - - - uri: /airs/collections/* - upstream: - nodes: - "airs-server:8000": 1 - plugins: - forward-auth: - uri: http://arlas-iam-server:9998/arlas_iam_server/auth - request_headers: ["Authorization", "arlas-org-filter"] - - - uri: /aproc/processes/* - upstream: - nodes: - "aproc-service:8001": 1 - plugins: - forward-auth: - uri: http://arlas-iam-server:9998/arlas_iam_server/auth - request_headers: ["Authorization", "arlas-org-filter"] - - - uri: /aproc/jobs - upstream: - nodes: - "aproc-service:8001": 1 - plugins: - forward-auth: - uri: http://arlas-iam-server:9998/arlas_iam_server/auth - request_headers: ["Authorization", "arlas-org-filter"] - - - uri: /aproc/jobs/* - upstream: - nodes: - "aproc-service:8001": 1 - plugins: - forward-auth: - uri: http://arlas-iam-server:9998/arlas_iam_server/auth - request_headers: ["Authorization", "arlas-org-filter"] -ssls: - - - snis: - - "localhost" - cert: | -${SSL_CERT} - key: | -${SSL_KEY} -#END diff --git a/conf/arlas.env b/conf/arlas.env index 600a16a..127d332 100644 --- a/conf/arlas.env +++ b/conf/arlas.env @@ -10,6 +10,68 @@ ARLAS_LOGGING_CONSOLE_LEVEL=INFO ARLAS_LOGGING_LEVEL=INFO ARLAS_SERVER_JDK_JAVA_OPTIONS +ARLAS_WUI_LINKS=' + [ + { + "name":"Hub", + "url":"/hub/", + "icon":"hub", + "check_url": "/arlas_persistence_server/healthcheck" + }, + { + "name":"IAM", + "url":"/iam/", + "icon":"manage_accounts", + "check_url": "/arlas_iam_server/healthcheck" + }, + { + "name": "Import", + "icon": "folder", + "url": "/fam-wui/", + "check_url": "/fam/healthcheck", + "check_url_response_type": "text" + } + ]' + +ARLAS_BUILDER_LINKS=' + [ + { + "name":"Hub", + "url":"/hub/", + "icon":"hub", + "check_url": "/arlas_persistence_server/healthcheck" + }, + { + "name":"IAM", + "url":"/iam/", + "icon":"manage_accounts", + "check_url": "/arlas_iam_server/healthcheck" + }, + { + "name": "Import", + "icon": "folder", + "url": "/fam-wui/", + "check_url": "/fam/healthcheck", + "check_url_response_type": "text" + } + ]' + +ARLAS_HUB_LINKS=' + [ + { + "name":"IAM", + "url":"/iam/", + "icon":"manage_accounts", + "check_url": "/arlas_iam_server/healthcheck" + }, + { + "name": "Import", + "icon": "folder", + "url": "/fam-wui/", + "check_url": "/fam/healthcheck", + "check_url_response_type": "text" + } + ]' #ARLAS_ANONYMOUS_VALUE= #ARLAS_APP_PATH= diff --git a/conf/arlas_iam.env b/conf/arlas_iam.env index d157e8f..7058e96 100644 --- a/conf/arlas_iam.env +++ b/conf/arlas_iam.env @@ -28,15 +28,23 @@ ARLAS_SERVER_URL=https://${ARLAS_HOST}/arlas #ARLAS_AUTH_PUBLIC_URIS="^(swagger.*:DELETE|swagger.*:GET|swagger.*:HEAD|swagger.*:OPTIONS|swagger.*:POST|swagger.*:PUT|swagger.*:PATCH)" ARLAS_AUTH_PUBLIC_URIS="swagger.*:*,stac:GET,openapi.json:GET,stac/.*:GET/POST,explore/.*:GET/POST,persist/.*:GET,collections/demo_.*:GET,authorize/resources:GET" -ARLAS_HUB_LINKS=' +ARLAS_IAM_LINKS=' [ { - "name":"Identity and Access", - "url":"/iam/", - "icon":"manage_accounts", - "check_url": "/arlas_iam_server/healthcheck" + "name":"Hub", + "url":"/hub/", + "icon":"hub", + "check_url": "/arlas_persistence_server/healthcheck" + }, + { + "name": "Import", + "icon": "folder", + "url": "/fam-wui/", + "check_url": "/fam/healthcheck", + "check_url_response_type": "text" } ]' + #ARLAS_AUTHENT_CLIENT_ID= #ARLAS_AUTHENT_DISABLE_AT_HASH_CHECK= #ARLAS_AUTHENT_ENABLE_SESSION_CHECKS= diff --git a/conf/elastic.env b/conf/elastic.env index 5af237b..ec136f4 100644 --- a/conf/elastic.env +++ b/conf/elastic.env @@ -9,7 +9,7 @@ FILEBEAT_ELASTIC_USERNAME= FILEBEAT_KIBANA_USERNAME= KIBANA_PASSWORD= METRICBEAT_CONF_DIRECTORY= -ELASTIC_STORAGE=arlas-test-data-es +ELASTIC_STORAGE=arlas-data-es #ELASTIC_APM_LOG_ECS_FORMATTER_ALLOW_LIST= #ELASTIC_APM_LOG_ECS_REFORMATTING= diff --git a/conf/minio.env b/conf/minio.env index d3302cf..16cbb3d 100644 --- a/conf/minio.env +++ b/conf/minio.env @@ -1,3 +1,4 @@ -MINIO_MC_CONF_DIR= -MINIO_ROOT_PASSWORD= -MINIO_ROOT_USER +MINIO_MC_CONF_DIR=arlas-data-mc-conf +MINIO_ROOT_PASSWORD=airssecret +# at least 8 characters +MINIO_ROOT_USER=airs diff --git a/conf/persistence-file.env b/conf/persistence-file.env index 3b9d434..e3c3994 100644 --- a/conf/persistence-file.env +++ b/conf/persistence-file.env @@ -3,7 +3,7 @@ ARLAS_PERSISTENCE_LOGGING_CONSOLE_LEVEL=INFO ARLAS_PERSISTENCE_LOGGING_LEVEL=INFO ARLAS_PERSISTENCE_PREFIX=/persist ARLAS_PERSISTENCE_URL=/persist -ARLAS_PERSISTENCE_STORAGE=arlas-test-persist +ARLAS_PERSISTENCE_STORAGE=arlas-persist #ARLAS_PERSISTENCE_APP_PATH= #ARLAS_AUTH_ENABLED= diff --git a/conf/postgres.env b/conf/postgres.env index 8d4c6c8..ed5e6ee 100644 --- a/conf/postgres.env +++ b/conf/postgres.env @@ -7,7 +7,7 @@ POSTGRES_DAY_OF_WEEK_TO_KEEP=6 POSTGRES_DAYS_TO_KEEP=7 POSTGRES_WEEKS_TO_KEEP=5 -POSTGRES_STORAGE=arlas-test-postgres +POSTGRES_STORAGE=arlas-postgres POSTGRES_BACKUP_STORAGE=/tmp/backup POSTGRES_CREATE_TABLE=${PWD}/conf/postgres/conf/pgCreateTable.sql POSTGRES_CRON=${PWD}/conf/postgres/cron/pg_backup_rotated.sh diff --git a/conf/versions.env b/conf/versions.env index 1def05f..c30e7d4 100644 --- a/conf/versions.env +++ b/conf/versions.env @@ -6,20 +6,20 @@ ARLAS_HUB_VERSION=gisaia/arlas-wui-hub:26.0.1 ARLAS_IAM_SERVER_VERSION=gisaia/arlas-iam-server:26.0.0 ARLAS_PERMISSIONS_VERSION=gisaia/arlas-permissions-server:26.0.0 ARLAS_PERSISTENCE_VERSION=gisaia/arlas-persistence-server:26.0.0 -ARLAS_SERVER_VERSION=gisaia/arlas-server:26.0.1 +ARLAS_SERVER_VERSION=gisaia/arlas-server:26.0.2 -ARLAS_VERSION_AIRS=gisaia/airs:0.4.11 -ARLAS_VERSION_APROC_PROC=gisaia/aproc-proc:0.4.11 -ARLAS_VERSION_APROC_SERVICE=gisaia/aproc-service:0.4.11 -ARLAS_VERSION_FAM_WUI=gisaia/arlas-fam-wui:0.4.11 -ARLAS_VERSION_FAM=gisaia/fam:0.4.11 +ARLAS_VERSION_AIRS=gisaia/airs:0.4.17 +ARLAS_VERSION_APROC_PROC=gisaia/aproc-proc:0.4.17 +ARLAS_VERSION_APROC_SERVICE=gisaia/aproc-service:0.4.17 +ARLAS_VERSION_FAM_WUI=gisaia/arlas-fam-wui:0.4.17 +ARLAS_VERSION_FAM=gisaia/fam:0.4.17 -ARLAS_VERSION_MINIO_MC=minio/mc:RELEASE.2024-04-29T09-56-05Z -ARLAS_VERSION_MINIO=minio/minio:RELEASE.2024-05-01T01-11-10Z +ARLAS_VERSION_MINIO_MC=minio/mc:RELEASE.2024-10-02T08-27-28Z +ARLAS_VERSION_MINIO=minio/minio:RELEASE.2024-10-02T17-50-41Z ARLAS_VERSION_RABBITMQ=rabbitmq:3.13.2-management-alpine ARLAS_VERSION_REDIS=redis/redis-stack:7.2.0-v10 -APISIX_VERSION=apache/apisix:3.8.0-debian +APISIX_VERSION=apache/apisix:3.9.1-debian ELASTIC_VERSION=docker.elastic.co/elasticsearch/elasticsearch:8.9.2 FILEBEAT_VERSION=docker.elastic.co/beats/filebeat:8.9.2 @@ -28,8 +28,6 @@ APM_VERSION=docker.elastic.co/apm/apm-server:8.9.2 KEYCLOAK_VERSION=quay.io/keycloak/keycloak:23.0 -MINIO_MC_VERSION=minio/mc:RELEASE.2024-03-20T21-07-29Z -MINIO_VERSION=minio/minio:RELEASE.2024-03-21T23-13-43Z PGADMIN4_VERSION=dpage/pgadmin4:7.8 POSTGRES_VERSION=postgres:16.1 diff --git a/dc/ref-dc-aias-aproc-proc.yaml b/dc/ref-dc-aias-aproc-proc.yaml index cedd0c9..d014b27 100644 --- a/dc/ref-dc-aias-aproc-proc.yaml +++ b/dc/ref-dc-aias-aproc-proc.yaml @@ -64,4 +64,4 @@ services: volumes: - ${APROC_INPUT_DIR}:/inputs:ro - ${APROC_DOWNLOAD_DIR}:/outbox - - ${PWD}/conf/aias/drivers.yaml:/app/conf/drivers.yaml:ro + - ${PWD}/conf/aias/drivers.yaml:/home/app/worker/conf/drivers.yaml:ro diff --git a/dc/ref-dc-aias-fam-wui.yaml b/dc/ref-dc-aias-fam-wui.yaml index 3be0385..bb85d4d 100644 --- a/dc/ref-dc-aias-fam-wui.yaml +++ b/dc/ref-dc-aias-fam-wui.yaml @@ -7,6 +7,8 @@ services: container_name: fam-wui restart: always depends_on: + arlas-iam-server: + condition: service_healthy arlas-server: condition: service_healthy fam-service: @@ -17,15 +19,18 @@ services: - ARLAS_AUTHENT_MODE=iam - ARLAS_IAM_SERVER_URL=https://${ARLAS_HOST}/arlas_iam_server - ARLAS_AUTHENT_THRESHOLD=60000 + - ARLAS_AUTHENT_SIGN_UP_ENABLED=false - ARLAS_TAB_NAME="ARLAS FAM Wui" - FAM_SERVER_URL=https://${ARLAS_HOST}/fam - FAM_DEFAULT_PATH='' - - FAM_COLLECTION=${AIAS_COLLECTION_NAME} + - FAM_COLLECTION=${AIRS_COLLECTION} + - FAM_ARCHIVES_PAGES_SIZE=${FAM_ARCHIVES_PAGES_SIZE} + - FAM_FILES_PAGES_SIZE=${FAM_FILES_PAGES_SIZE} - APROC_SERVER_URL=https://${ARLAS_HOST}/aproc - - APROC_COLLECTION=${AIAS_COLLECTION_NAME} + - APROC_COLLECTION=${AIRS_COLLECTION} - APROC_CATALOG=${AIAS_CATALOG_NAME} - AIRS_SERVER_URL=https://${ARLAS_HOST}/airs - - AIRS_COLLECTION=${AIAS_COLLECTION_NAME} + - AIRS_COLLECTION=${AIRS_COLLECTION} - 'ARLAS_STATIC_LINKS=${ARLAS_FAM_LINKS}' expose: - "80" diff --git a/dc/ref-dc-aias-fam.yaml b/dc/ref-dc-aias-fam.yaml index c1ecb45..23aeb89 100644 --- a/dc/ref-dc-aias-fam.yaml +++ b/dc/ref-dc-aias-fam.yaml @@ -13,8 +13,11 @@ services: - APROC_RESOURCE_ID_HASH_STARTS_AT=3 volumes: - ${APROC_INPUT_DIR}:/inputs:ro + - ${PWD}/conf/aias/drivers.yaml:/app/conf/drivers.yaml:ro expose: - "8005" + ports: + - 8005:8005 networks: - arlas-net healthcheck: diff --git a/dc/ref-dc-aias-minio-init.yaml b/dc/ref-dc-aias-minio-init.yaml index 7f83767..91dbd73 100644 --- a/dc/ref-dc-aias-minio-init.yaml +++ b/dc/ref-dc-aias-minio-init.yaml @@ -10,7 +10,7 @@ services: - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - MINIO_ROOT_USER=${MINIO_ROOT_USER} volumes: - - ${MINIO_MC_CONF_DIR:-/tmp}:/root + - ${MINIO_MC_CONF_DIR:-arlas-data-mc-conf}:/root networks: - arlas-net logging: diff --git a/dc/ref-dc-aias-minio.yaml b/dc/ref-dc-aias-minio.yaml index e912659..a970c8d 100644 --- a/dc/ref-dc-aias-minio.yaml +++ b/dc/ref-dc-aias-minio.yaml @@ -13,7 +13,7 @@ services: networks: - arlas-net volumes: - - ${AIRS_STORAGE_DIRECTORY:-/tmp/minio}:/data + - ${AIRS_STORAGE_DIRECTORY:-arlas-data-minio}:/data logging: driver: "${DOCKER_LOGGING_DRIVER:-json-file}" options: diff --git a/dc/ref-dc-arlas-builder.yaml b/dc/ref-dc-arlas-builder.yaml index 1dff70b..bf12d1d 100644 --- a/dc/ref-dc-arlas-builder.yaml +++ b/dc/ref-dc-arlas-builder.yaml @@ -38,6 +38,7 @@ services: - ARLAS_SERVER_URL=${ARLAS_SERVER_URL:-/arlas} - ARLAS_USE_AUTHENT=${ARLAS_USE_AUTHENT} - ARLAS_WUI_URL=${ARLAS_WUI_URL:-/wui/} + - 'ARLAS_STATIC_LINKS=${ARLAS_BUILDER_LINKS}' expose: - "80" networks: diff --git a/dc/ref-dc-arlas-hub.yaml b/dc/ref-dc-arlas-hub.yaml index 7db8dce..d19b391 100644 --- a/dc/ref-dc-arlas-hub.yaml +++ b/dc/ref-dc-arlas-hub.yaml @@ -33,6 +33,7 @@ services: - ARLAS_PERSISTENCE_URL=${ARLAS_PERSISTENCE_URL:-/persist} - ARLAS_USE_AUTHENT=${ARLAS_USE_AUTHENT} - ARLAS_WUI_URL=${ARLAS_WUI_URL:-/wui/} + - 'ARLAS_STATIC_LINKS=${ARLAS_HUB_LINKS}' expose: - "80" networks: diff --git a/dc/ref-dc-arlas-wui.yaml b/dc/ref-dc-arlas-wui.yaml index e99afff..05ece1e 100644 --- a/dc/ref-dc-arlas-wui.yaml +++ b/dc/ref-dc-arlas-wui.yaml @@ -39,6 +39,7 @@ services: - ARLAS_USE_AUTHENT=${ARLAS_USE_AUTHENT} - ARLAS_WUI_BASE_HREF=${ARLAS_WUI_BASE_HREF:-/wui} - PUBLIC_HOST=${ARLAS_HOST} + - 'ARLAS_STATIC_LINKS=${ARLAS_WUI_LINKS}' expose: - "80" networks: diff --git a/dc/ref-dc-elastic-init.yaml b/dc/ref-dc-elastic-init.yaml index cfc542d..f74b591 100644 --- a/dc/ref-dc-elastic-init.yaml +++ b/dc/ref-dc-elastic-init.yaml @@ -78,7 +78,7 @@ services: echo "Create backup SLM policy"; curl -s -X PUT --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" \ -H "Content-Type: application/json" https://elasticsearch:9200/_slm/policy/nightly-snapshots \ - -d "{\"schedule\":\"0 30 1 * * ?\",\"name\":\"\",\"repository\":\"backup\",\"config\":{\"indices\":\"${AIRS_INDEX_COLLECTION_PREFIX}_${AIAS_COLLECTION_NAME},.arlas,${APROC_INDEX_NAME}\",\"include_global_state\":true},\"retention\":{\"expire_after\":\"1m\",\"min_count\":5,\"max_count\":10}}" + -d "{\"schedule\":\"0 30 1 * * ?\",\"name\":\"\",\"repository\":\"backup\",\"config\":{\"indices\":\"${AIRS_INDEX_COLLECTION_PREFIX}_*,.arlas,${APROC_INDEX_NAME}\",\"include_global_state\":true},\"retention\":{\"expire_after\":\"1m\",\"min_count\":5,\"max_count\":10}}" echo "" echo "Configure SLM retention task"; curl -s -X PUT --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" \ diff --git a/dc/ref-dc-iam-server.yaml b/dc/ref-dc-iam-server.yaml index 2e0ead0..7762eb1 100644 --- a/dc/ref-dc-iam-server.yaml +++ b/dc/ref-dc-iam-server.yaml @@ -1,5 +1,5 @@ services: - auth-server: # ARLAS IAM is the ARLAS Identity and Access Management service. + arlas-iam-server: # ARLAS IAM is the ARLAS Identity and Access Management service. image: ${ARLAS_IAM_SERVER_VERSION} container_name: arlas-iam-server depends_on: diff --git a/dc/ref-dc-iam-wui.yaml b/dc/ref-dc-iam-wui.yaml index 61d7add..d46485d 100644 --- a/dc/ref-dc-iam-wui.yaml +++ b/dc/ref-dc-iam-wui.yaml @@ -5,7 +5,7 @@ services: depends_on: arlas-server: condition: service_healthy - auth-server: + arlas-iam-server: condition: service_healthy restart: ${ARLAS_WUI_IAM_RESTART_STRATEGY:-always} environment: @@ -16,6 +16,7 @@ services: - ARLAS_USE_AUTHENT=${ARLAS_USE_AUTHENT} - ARLAS_WUI_IAM_APP_PATH=/iam - ARLAS_WUI_IAM_BASE_HREF=/iam + - 'ARLAS_STATIC_LINKS=${ARLAS_IAM_LINKS}' expose: - "80" networks: diff --git a/dc/ref-dc-volumes.yaml b/dc/ref-dc-volumes.yaml index 8a63d8d..5a6430e 100644 --- a/dc/ref-dc-volumes.yaml +++ b/dc/ref-dc-volumes.yaml @@ -1,18 +1,18 @@ volumes: # Elasticsearch - arlas-test-data-es: - name: arlas-test-data-es - arlas-test-data-es-logs: - name: arlas-test-data-es-logs - arlas-test-es-certs: - name: arlas-test-es-certs - arlas-test-es-config: - name: arlas-test-es-config + arlas-data-es: + name: arlas-data-es + arlas-data-es-logs: + name: arlas-data-es-logs + arlas-es-certs: + name: arlas-es-certs + arlas-es-config: + name: arlas-es-config # Persistence - arlas-test-persist: - name: arlas-test-persist + arlas-persist: + name: arlas-persist # Postgres - arlas-test-postgres: - name: arlas-test-postgres \ No newline at end of file + arlas-postgres: + name: arlas-postgres \ No newline at end of file diff --git a/scripts/create_aias_collection.sh b/scripts/create_aias_collection.sh new file mode 100755 index 0000000..afc2ce1 --- /dev/null +++ b/scripts/create_aias_collection.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -o errexit -o pipefail +. conf/aias.env +ORG=org.com +[ -z "$1" ] && echo "Please provide the ARLAS configuration name" && exit 1; +[ -z "$2" ] && echo "Please provide the collection name" && exit 1; +[ -z "$3" ] && echo "Please provide the index name" && exit 2; +[ ! -z "$4" ] && ORG=$4 ; + +CONF=$1 +COLLECTION=$2 +INDEX=$3 + +GROUPS_PARAMS='--reader group/config.json/org.com --writer group/config.json/org.com' +USER_CONF="local.iam.user" + +echo "Create collection '${COLLECTION}' on index '${INDEX}'" +arlas_cli --config-file /tmp/arlas-cli.yaml \ + collections --config ${CONF} create ${COLLECTION} \ + --index ${INDEX} --display-name ${COLLECTION} \ + --id-path id \ + --centroid-path centroid \ + --geometry-path geometry \ + --date-path properties.datetime \ + --no-public \ + --owner ${ORG} \ + --orgs ${ORG} diff --git a/scripts/generate_apisix_conf.sh b/scripts/generate_apisix_conf.sh index ef79ffc..a70389c 100755 --- a/scripts/generate_apisix_conf.sh +++ b/scripts/generate_apisix_conf.sh @@ -1,10 +1,5 @@ #!/usr/bin/env bash set -o errexit -o pipefail -if [ -z "$1" ] -then - echo "Error: no apisix configuration file provided" - exit 0 -fi if [ ! -f conf/server.crt ] || [ ! -f conf/server.key ] then diff --git a/scripts/reset_data.sh b/scripts/reset_data.sh index d7a01b1..9eb6683 100755 --- a/scripts/reset_data.sh +++ b/scripts/reset_data.sh @@ -1,3 +1,2 @@ #!/bin/bash - -docker volume rm arlas-test-data-es arlas-test-persist arlas-test-postgres +docker volume rm arlas-test-data-es arlas-test-persist arlas-test-postgres arlas-test-data-minio diff --git a/start.sh b/start.sh index a285a6a..be191b7 100755 --- a/start.sh +++ b/start.sh @@ -3,7 +3,7 @@ set -o errexit -o pipefail COMPOSE_FILES=" -f dc/ref-dc-volumes.yaml -f dc/ref-dc-arlas-server.yaml -f dc/ref-dc-elastic.yaml -f dc/ref-dc-arlas-persistence-server.yaml -f dc/ref-dc-arlas-permissions-server.yaml -f dc/ref-dc-arlas-builder.yaml -f dc/ref-dc-arlas-hub.yaml -f dc/ref-dc-arlas-wui.yaml -f dc/ref-dc-protomaps.yaml -f dc/ref-dc-net.yaml" COMPOSE_SERVICES="elasticsearch arlas-server arlas-persistence-server arlas-permissions-server arlas-builder arlas-hub arlas-wui protomaps apisix" -ENV_FILES="--env-file conf/versions.env --env-file conf/elastic.env --env-file conf/arlas.env --env-file conf/persistence-file.env --env-file conf/permissions.env --env-file conf/apisix.env --env-file conf/restart_strategy.env --env-file conf/stack.env" +ENV_FILES="conf/versions.env conf/elastic.env conf/arlas.env conf/persistence-file.env conf/permissions.env conf/apisix.env conf/restart_strategy.env conf/stack.env" if [ -z "$1" ] then @@ -16,37 +16,50 @@ fi if [ "$1" = "iam" ] then echo "START STACK WITH IAM" - ./scripts/generate_apisix_conf.sh conf/apisix/apisix_with_iam.yaml COMPOSE_FILES=${COMPOSE_FILES}" -f dc/ref-dc-iam-wui.yaml -f dc/ref-dc-apisix-ssl.yaml -f dc/ref-dc-iam-server.yaml -f dc/ref-dc-postgres.yaml" - COMPOSE_SERVICES=${COMPOSE_SERVICES}" auth-server arlas-wui-iam db" - ENV_FILES=${ENV_FILES}" --env-file conf/arlas_iam.env --env-file conf/postgres.env" - + COMPOSE_SERVICES=${COMPOSE_SERVICES}" arlas-iam-server arlas-wui-iam db" + ENV_FILES=${ENV_FILES}" conf/arlas_iam.env conf/postgres.env" cat conf/apisix/apisix_part_arlas_services.yaml > conf/apisix/apisix.yaml cat conf/apisix/apisix_part_iam_services.yaml >> conf/apisix/apisix.yaml cat conf/apisix/apisix_part_ssl.yaml >> conf/apisix/apisix.yaml echo "#END" >> conf/apisix/apisix.yaml + ./scripts/generate_apisix_conf.sh conf/apisix/apisix.yaml fi if [ "$1" = "aias" ] then echo "START STACK WITH AIAS AND IAM" - ./scripts/generate_apisix_conf.sh conf/apisix/apisix_with_aias.yaml - COMPOSE_FILES=${COMPOSE_FILES}" -f dc/ref-dc-arlas-server-iam.yaml -f dc/ref-dc-apisix-ssl.yaml -f dc/ref-dc-aias-volumes.yaml -f dc/ref-dc-aias-airs.yaml -f dc/ref-dc-aias-aproc-service.yaml -f dc/ref-dc-aias-aproc-proc.yaml -f dc/ref-dc-aias-minio-init.yaml -f dc/ref-dc-aias-minio.yaml -f dc/ref-dc-aias-redis.yaml -f dc/ref-dc-aias-rabbitmq.yaml -f dc/ref-dc-aias-fam.yaml -f dc/ref-dc-aias-fam-wui.yaml" - COMPOSE_FILES=${COMPOSE_FILES}" -f dc/ref-dc-aias-airs.yaml -f dc/ref-dc-aias-aproc-proc.yaml -f dc/ref-dc-aias-aproc-service.yaml -f dc/ref-dc-aias-fam-wui.yaml -f dc/ref-dc-aias-fam.yaml -f dc/ref-dc-aias-minio-init.yaml -f dc/ref-dc-aias-minio.yaml -f dc/ref-dc-aias-rabbitmq.yaml -f dc/ref-dc-aias-redis.yaml -f dc/ref-dc-aias-volumes.yaml" - COMPOSE_SERVICES=${COMPOSE_SERVICES}" auth-server arlas-wui-iam db" - COMPOSE_SERVICES=${COMPOSE_SERVICES}" airs-server aproc-service aproc-proc redis rabbitmq fam-service arlas-fam-wui" - ENV_FILES=${ENV_FILES}" --env-file conf/arlas_iam.env --env-file conf/postgres.env" - ENV_FILES=${ENV_FILES}" --env-file conf/aias.env" + COMPOSE_FILES=${COMPOSE_FILES}" -f dc/ref-dc-iam-wui.yaml -f dc/ref-dc-apisix-ssl.yaml -f dc/ref-dc-iam-server.yaml -f dc/ref-dc-postgres.yaml" + COMPOSE_FILES=${COMPOSE_FILES}" -f dc/ref-dc-aias-airs.yaml -f dc/ref-dc-aias-aproc-proc.yaml -f dc/ref-dc-aias-aproc-service.yaml -f dc/ref-dc-aias-fam-wui.yaml -f dc/ref-dc-aias-fam.yaml -f dc/ref-dc-aias-minio.yaml -f dc/ref-dc-aias-rabbitmq.yaml -f dc/ref-dc-aias-redis.yaml -f dc/ref-dc-aias-volumes.yaml" + COMPOSE_SERVICES=${COMPOSE_SERVICES}" arlas-iam-server arlas-wui-iam db" + COMPOSE_SERVICES=${COMPOSE_SERVICES}" airs-server aproc-service aproc-proc redis rabbitmq fam-service arlas-fam-wui minio" + ENV_FILES=${ENV_FILES}" conf/arlas_iam.env conf/postgres.env" + ENV_FILES=${ENV_FILES}" conf/aias.env conf/minio.env" cat conf/apisix/apisix_part_arlas_services.yaml > conf/apisix/apisix.yaml cat conf/apisix/apisix_part_iam_services.yaml >> conf/apisix/apisix.yaml cat conf/apisix/apisix_part_aias_services.yaml >> conf/apisix/apisix.yaml cat conf/apisix/apisix_part_ssl.yaml >> conf/apisix/apisix.yaml echo "#END" >> conf/apisix/apisix.yaml + ./scripts/generate_apisix_conf.sh conf/apisix/apisix.yaml + + echo "Initialising Minio configuration..." + set +e + docker compose -p arlas-exploration-stack \ + --env-file conf/versions.env \ + --env-file conf/stack.env \ + --env-file conf/aias.env \ + --env-file conf/minio.env \ + -f dc/ref-dc-net.yaml -f dc/ref-dc-aias-minio-init.yaml -f dc/ref-dc-aias-minio.yaml -f dc/ref-dc-aias-volumes.yaml -f dc/ref-dc-volumes.yaml \ + up -d --wait --wait-timeout 300 minio createbuckets + echo "...done." + set -e fi # We run elastic on 9200 without ssl # set +e # docker compose -p arlas-exploration-stack $ENV_FILES -f dc/ref-dc-elastic-init.yaml -f dc/ref-dc-elastic.yaml -f dc/ref-dc-net.yaml up -d --wait --wait-timeout 300 # set -e -docker compose -p arlas-exploration-stack $ENV_FILES $COMPOSE_FILES up -d --remove-orphans --wait --wait-timeout 300 $COMPOSE_SERVICES + +cat ${ENV_FILES} > docker-compose.env +docker compose -p arlas-exploration-stack --env-file docker-compose.env $COMPOSE_FILES up -d --remove-orphans --wait --wait-timeout 300 $COMPOSE_SERVICES