basic stack

.gitignore

@@ -1,2 +1,8 @@
 .env
-.DS_Store
+.DS_Store
+sample/sample.json
+conf/server.crt
+conf/server.key
+conf/domains.generated.ext
+conf/apisix.generated.yaml
+conf/apisix.generated.tmp.yaml

conf/aias.env

@@ -0,0 +1,61 @@
+AIRS_CATALOG=catalog
+AIRS_COLLECTION=main
+AIRS_INDEX_ENDPOINT_URL=http://elasticsearch:9200
+AIRS_LOGGER_LEVEL=INFO
+AIRS_S3_ACCESS_KEY_ID=airs
+AIRS_S3_ASSET_HTTP_ENDPOINT_URL=http://minio:9000/{}/{}
+AIRS_S3_BUCKET=airs
+AIRS_S3_SECRET_ACCESS_KEY=airssecret
+AIRS_STORAGE_DIRECTORY=
+AIRS_S3_ENDPOINT_URL=http://minio:9000
+
+APROC_DOWNLOAD_DIR=/tmp/outbox
+APROC_INPUT_DIR=/tmp/inputs
+APROC_CONFIGURATION_FILE=`pwd`/conf/aias/aproc.yaml
+APROC_CORS_ORIGINS="*"
+APROC_CORS_METHODS="*"
+APROC_CORS_HEADERS="*"
+ARLAS_SMTP_ACTIVATED=false
+ARLAS_SMTP_HOST=tobechanged
+ARLAS_SMTP_PORT=25
+ARLAS_SMTP_USERNAME=tobechanged
+ARLAS_SMTP_PASSWORD=tobechanged
+ARLAS_SMTP_FROM=[email protected]
+APROC_DOWNLOAD_OUTBOX_DIR=/outbox
+APROC_DOWNLOAD_ADMIN_EMAILS="[email protected],[email protected]"
+
+APROC_DOWNLOAD_SUBJECT_USER="\"ARLAS Services: Your download of {collection}/{item_id} is available.\""
+APROC_DOWNLOAD_CONTENT_USER="\"ARLAS Services: Dear {arlas-user-email}. <br>Your download of {collection}/{item_id} is available for projection {target_projection} ({target_format}). <br>ARLAS Services.\""
+
+APROC_DOWNLOAD_REQUEST_SUBJECT_USER="\"ARLAS Services: Thank you for your download request (({collection}/{item_id}).\""
+APROC_DOWNLOAD_REQUEST_CONTENT_USER="\"ARLAS Services: Dear {arlas-user-email}. <br>Your download request for {collection}/{item_id} with projection {target_projection} ({target_format}) will shortly be taken into account. <br>ARLAS Services.\""
+
+APROC_DOWNLOAD_SUBJECT_ERROR="\"ARLAS Services: ERROR: The download of {collection}/{item_id} failed.\""
+APROC_DOWNLOAD_CONTENT_ERROR="\"ARLAS Services: The download of {collection}/{item_id} failed ({error}).\""
+
+APROC_DOWNLOAD_SUBJECT_ADMIN="\"ARLAS Services: The download of {collection}/{item_id} for {arlas-user-email} is available.\""
+APROC_DOWNLOAD_CONTENT_ADMIN="\"ARLAS Services: The download of {collection}/{item_id} for {arlas-user-email} is available in {target_directory} ({file_name}) for projection {target_projection} ({target_format}). <br>ARLAS Services.\""
+
+APROC_DOWNLOAD_REQUEST_SUBJECT_ADMIN="\"ARLAS Services: {arlas-user-email} requested the download of {collection}/{item_id}.\""
+APROC_DOWNLOAD_REQUEST_CONTENT_ADMIN="\"ARLAS Services: {arlas-user-email} requested the download of {collection}/{item_id} for projection {target_projection} ({target_format}). <br>ARLAS Services.\""
+
+APROC_EMAIL_PATH_PREFIX_ADD="/tmp/"
+APROC_PATH_TO_WINDOWS=false
+APROC_LOGGER_LEVEL=INFO
+APROC_ENDPOINT_FROM_APROC=http://aproc-service:8001/arlas/aproc
+APROC_INDEX_ENDPOINT_URL=http://elasticsearch:9200
+APROC_INDEX_NAME=aproc_downloads
+APROC_RESOURCE_ID_HASH_STARTS_AT=1
+ARLAS_URL_SEARCH="http://arlas-server:9999/arlas/explore/{collection}/_search?f=id:eq:{item}"
+
+ARLAS_HOST=localhost
+
+APISIX_CONF_FILE=${PWD}/conf/apisix/apisix.generated.yaml
+
+#AIRS_HOST=
+#AIRS_INDEX_COLLECTION_PREFIX=
+#AIRS_MAPPING_URL=
+#AIRS_PORT=
+#AIRS_PREFIX=
+#AIRS_S3_PLATFORM=
+#AIRS_S3_TIER=

conf/aias/airs.yaml

@@ -0,0 +1,23 @@
+index:
+  endpoint_url: $AIRS_INDEX_ENDPOINT_URL|http://localhost:9200
+  collection_prefix: $AIRS_INDEX_COLLECTION_PREFIX|airs
+  login: $AIRS_INDEX_LOGIN
+  pwd: $AIRS_INDEX_PWD
+
+s3:
+  bucket: $AIRS_S3_BUCKET|arlaseo
+  access_key_id: $AIRS_S3_ACCESS_KEY_ID
+  secret_access_key: $AIRS_S3_SECRET_ACCESS_KEY
+  region: $AIRS_S3_REGION|None
+  tier: $AIRS_S3_TIER|"Standard"
+  platform: $AIRS_S3_PLATFORM
+
+# asset_http_endpoint_url must look like https://storage.googleapis.com/{}/{}
+# first element is the bucket, second is the path to the object
+  asset_http_endpoint_url: $AIRS_S3_ASSET_HTTP_ENDPOINT_URL
+
+# if you don't use aws s3, then please prov
+  endpoint_url: $AIRS_S3_ENDPOINT_URL
+
+arlaseo_mapping_url: $AIRS_MAPPING_URL|"https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json"
+arlaseo_collection_url: $AIRS_COLLECTION_URL|"https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/collection.json"

conf/aias/aproc.yaml

@@ -0,0 +1,19 @@
+celery_broker_url: $CELERY_BROKER_URL|pyamqp://guest:[email protected]:5672//
+celery_result_backend: $CELERY_RESULT_BACKEND|redis://127.0.0.1:6379/0
+airs_endpoint: $AIRS_ENDPOINT|http://127.0.0.1:8000/arlas/airs
+processes:
+  -
+    name: ingest
+    class_name: extensions.aproc.proc.ingest.ingest_process
+    configuration:
+      drivers: conf/drivers.yaml
+  -
+    name: directory_ingest
+    class_name: extensions.aproc.proc.ingest.directory_ingest_process
+    configuration:
+      drivers: conf/drivers.yaml
+  -
+    name: download
+    class_name: extensions.aproc.proc.download.download_process
+    configuration:
+      drivers: conf/download_drivers.yaml

conf/aias/download_drivers.yaml

@@ -0,0 +1,54 @@
+arlas_url_search: $ARLAS_URL_SEARCH
+arlaseo_mapping_url: $ARLASEO_MAPPING_URL|"https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/mapping.json"
+download_mapping_url: $DOWNLOAD_MAPPING_URL|"https://raw.githubusercontent.com/gisaia/ARLAS-EO/v0.0.6/downloads_mapping.json"
+
+outbox_directory: $APROC_DOWNLOAD_OUTBOX_DIR
+notification_admin_emails: $APROC_DOWNLOAD_ADMIN_EMAILS
+
+# Usable variables in messages: {target_projection}, {target_format}, {arlas-user-email}, {item_id}, {asset_name}, {collection}, {target_directory}, {file_name}
+# Usable variables in messages: {error}
+
+email_subject_user: $APROC_DOWNLOAD_SUBJECT_USER
+email_content_user: $APROC_DOWNLOAD_CONTENT_USER
+email_subject_error_download: $APROC_DOWNLOAD_SUBJECT_ERROR
+email_content_error_download: $APROC_DOWNLOAD_CONTENT_ERROR
+email_subject_admin: $APROC_DOWNLOAD_SUBJECT_ADMIN
+email_content_admin: $APROC_DOWNLOAD_CONTENT_ADMIN
+
+email_request_subject_user: $APROC_DOWNLOAD_REQUEST_SUBJECT_USER
+email_request_content_user: $APROC_DOWNLOAD_REQUEST_CONTENT_USER
+email_request_subject_admin: $APROC_DOWNLOAD_REQUEST_SUBJECT_ADMIN
+email_request_content_admin: $APROC_DOWNLOAD_REQUEST_CONTENT_ADMIN
+
+email_path_prefix_add: $APROC_EMAIL_PATH_PREFIX_ADD
+email_path_to_windows: $APROC_PATH_TO_WINDOWS
+
+smtp:
+  host: $ARLAS_SMTP_HOST
+  port: $ARLAS_SMTP_PORT
+  login: $ARLAS_SMTP_USERNAME
+  password: $ARLAS_SMTP_PASSWORD
+  from_addr: $ARLAS_SMTP_FROM
+
+index_for_download:
+  endpoint_url: $APROC_INDEX_ENDPOINT_URL|http://localhost:9200
+  index_name: $APROC_INDEX_NAME|aproc_downloads
+  login: $APROC_INDEX_LOGIN
+  pwd: $APROC_INDEX_PWD
+
+drivers:
+  -
+    name: simple_copy
+    class_name: extensions.aproc.proc.download.drivers.impl.simple_copy
+    configuration:
+    priority: 3
+  -
+    name: met_file
+    class_name: extensions.aproc.proc.download.drivers.impl.met_file
+    configuration:
+    priority: 1
+  -
+    name: image_file
+    class_name: extensions.aproc.proc.download.drivers.impl.image_file
+    configuration:
+    priority: 2

conf/aias/drivers.yaml

@@ -0,0 +1,85 @@
+inputs_directory: $INGESTED_FOLDER|/inputs
+
+max_number_of_archive_for_ingest: 1000000
+aproc_endpoint: $APROC_ENDPOINT_FROM_APROC|http://localhost:8001
+resource_id_hash_starts_at: $APROC_RESOURCE_ID_HASH_STARTS_AT|1
+
+drivers:
+  -
+    name: dimap
+    class_name: extensions.aproc.proc.ingest.drivers.impl.dimap
+    assets_dir: /tmp/aproc/dimap
+    configuration:
+    priority: 1
+  -
+    name: geoeye
+    class_name: extensions.aproc.proc.ingest.drivers.impl.geoeye
+    assets_dir: /tmp/aproc/geoeye
+    configuration:
+    priority: 2
+  -
+    name: digitalglobe
+    class_name: extensions.aproc.proc.ingest.drivers.impl.digitalglobe
+    assets_dir: /tmp/aproc/digitalglobe
+    configuration:
+    priority: 3
+  -
+    name: ast_dem
+    class_name: extensions.aproc.proc.ingest.drivers.impl.ast_dem
+    assets_dir: /tmp/aproc/ast_dem
+    configuration:
+    priority: 4
+  -
+    name: rapideye
+    class_name: extensions.aproc.proc.ingest.drivers.impl.rapideye
+    assets_dir: /tmp/aproc/rapideye
+    configuration:
+    priority: 5
+  -
+    name: spot5
+    class_name: extensions.aproc.proc.ingest.drivers.impl.spot5
+    assets_dir: /tmp/aproc/spot5
+    configuration:
+    priority: 6
+  -
+    name: terrasarx
+    class_name: extensions.aproc.proc.ingest.drivers.impl.terrasarx
+    assets_dir: /tmp/aproc/terrasarx
+    configuration:
+      tmp_directory: $TMP_FOLDER|/tmp
+    priority: 7
+  -
+    name: cosmoskymed
+    class_name: extensions.aproc.proc.ingest.drivers.impl.cosmoskymed
+    assets_dir: /tmp/aproc/cosmoskymed
+    configuration:
+      tmp_directory: $TMP_FOLDER|/tmp
+    priority: 8
+
+
+  -
+    name: theia
+    class_name: extensions.aproc.proc.ingest.drivers.impl.theia
+    assets_dir: /tmp/aproc/theia
+    configuration:
+      token_url: https://theia.cnes.fr/atdistrib/services/authenticate/
+      login: $THEIA_LOGIN
+      pwd: $THEIA_PWD
+      manage_data: false
+    priority: 7
+
+  # -
+  #   name: tiff
+  #   class_name: extensions.aproc.proc.ingest.drivers.impl.tiff
+  #   assets_dir: /tmp/aproc/tiff
+  #   configuration:
+  #     tmp_directory: $TMP_FOLDER|/tmp
+  #   priority: 9
+
+  # -
+  #   name: jpeg2000
+  #   class_name: extensions.aproc.proc.ingest.drivers.impl.jpeg2000
+  #   assets_dir: /tmp/aproc/jpeg2000
+  #   configuration:
+  #     tmp_directory: $TMP_FOLDER|/tmp
+  #   priority: 10

conf/apisix.env

@@ -0,0 +1,3 @@
+APISIX_STAND_ALONE=true
+APISIX_CONF_DIRECTORY=
+APISIX_CONF_FILE=${PWD}/conf/apisix/apisix.yaml

conf/apisix/apisix.yaml

@@ -0,0 +1,71 @@
+routes:
+  -
+    uri: /builder
+    upstream:
+      nodes:
+        "arlas-builder:80": 1
+  -
+    uri: /builder/*
+    upstream:
+      nodes:
+        "arlas-builder:80": 1
+    plugins:
+      proxy-rewrite:
+        regex_uri: ["/builder/(.*)", "/$1"]
+  -
+    uri: /
+    plugins:
+      redirect:
+        uri: /hub/
+  -
+    uri: /hub
+    upstream:
+      nodes:
+        "arlas-hub:80": 1
+  -
+    uri: /hub/*
+    upstream:
+      nodes:
+        "arlas-hub:80": 1
+    plugins:
+      proxy-rewrite:
+        regex_uri: ["/hub/(.*)", "/$1"]
+  -
+    uri: /iam
+    upstream:
+      nodes:
+        "arlas-wui-iam:80": 1
+  -
+    uri: /iam/*
+    upstream:
+      nodes:
+        "arlas-wui-iam:80": 1
+  -
+    uri: /wui
+    upstream:
+      nodes:
+        "arlas-wui:80": 1
+  -
+    uri: /wui/*
+    upstream:
+      nodes:
+        "arlas-wui:80": 1
+    plugins:
+      proxy-rewrite:
+        regex_uri: ["/wui/(.*)", "/$1"]
+  -
+    uri: /arlas_permissions_server/*
+    upstream:
+      nodes:
+        "arlas-permissions-server:9996": 1
+  -
+    uri: /persist/*
+    upstream:
+      nodes:
+        "arlas-persistence-server:9997": 1
+  -
+    uri: /arlas/*
+    upstream:
+      nodes:
+        "arlas-server:9999": 1
+#END