From 72787f8d17024e67213ad9463bb48ca3c1239f5b Mon Sep 17 00:00:00 2001 From: Yury Hrytsuk <50014626+YuryHrytsuk@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:08:16 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Clean=20up=20simcore=20web?= =?UTF-8?q?server=20environment=20(#4491)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env-devel | 143 +++++++- .env-wb-db-event-listener | 38 -- .env-wb-garbage-collector | 38 -- services/docker-compose.devel.yml | 3 +- services/docker-compose.yml | 325 ++++++++++++++++-- .../application_settings.py | 4 +- .../application_settings_utils.py | 4 +- .../garbage_collector/settings.py | 5 - .../login/settings.py | 8 - .../resource_manager/settings.py | 5 - .../studies_dispatcher/settings.py | 1 - .../data/default_app_config-integration.yaml | 10 +- .../web/server/tests/integration/conftest.py | 7 +- .../isolated/test_diagnostics_healthcheck.py | 2 +- ..._studies_dispatcher_projects_permalinks.py | 2 +- .../tests/unit/with_dbs/03/login/conftest.py | 2 +- .../tests/unit/with_dbs/03/test_email.py | 2 +- tests/e2e/Makefile | 4 +- 18 files changed, 452 insertions(+), 151 deletions(-) delete mode 100644 .env-wb-db-event-listener delete mode 100644 .env-wb-garbage-collector diff --git a/.env-devel b/.env-devel index 61a44fd4410..cfd7401c6a8 100644 --- a/.env-devel +++ b/.env-devel @@ -81,8 +81,6 @@ S3_ENDPOINT=172.17.0.1:9001 S3_SECRET_KEY=12345678 S3_SECURE=0 - - SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key @@ -105,19 +103,83 @@ TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack # NOTE: WEBSERVER_SESSION_SECRET_KEY = $(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())") + +WEBSERVER_LOGLEVEL=INFO + +WEBSERVER_PROJECTS={} PROJECTS_MAX_COPY_SIZE_BYTES=30Gib PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES=5 + WEBSERVER_ANNOUNCEMENTS=1 + WEBSERVER_DEV_FEATURES_ENABLED=0 + WEBSERVER_HOST=webserver -WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 -WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=0 + +WEBSERVER_CATALOG={} +CATALOG_HOST=catalog +CATALOG_PORT=8000 + +WEBSERVER_SCICRUNCH={} + +WEBSERVER_SESSION_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key=' + +WEBSERVER_STATICWEB={} + +WEBSERVER_STUDIES_DISPATCHER={} +STUDIES_ACCESS_ANONYMOUS_ALLOWED=0 +STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png + +WEBSERVER_TRACING={} + +WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180 + +WEBSERVER_EXPORTER={} + +WEBSERVER_EMAIL={} + +DIRECTOR_HOST=director +DIRECTOR_PORT=8080 + +DIRECTOR_V2_HOST=director-v2 +DIRECTOR_V2_PORT=8000 + +WEBSERVER_DIAGNOSTICS={} +DIAGNOSTICS_MAX_AVG_LATENCY=10 +DIAGNOSTICS_MAX_TASK_DELAY=30 +DIAGNOSTICS_SLOW_DURATION_SECS=1 + +WEBSERVER_GARBAGE_COLLECTOR=null + +WEBSERVER_DB_LISTENER=0 + +WEBSERVER_LOGIN={} +LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 +LOGIN_REGISTRATION_INVITATION_REQUIRED=0 +LOGIN_2FA_REQUIRED=0 +LOGIN_2FA_CODE_EXPIRATION_SEC=60 + +WEBSERVER_FRONTEND={} + +WEBSERVER_ACTIVITY=null WEBSERVER_PROMETHEUS_API_VERSION=v1 WEBSERVER_PROMETHEUS_URL=http://prometheus:9090 -WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=900 -WEBSERVER_SESSION_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key=' -WEBSERVER_STUDIES_ACCESS_ENABLED=0 +RESOURCE_MANAGER_RESOURCE_TTL_S=900 + +RESOURCE_USAGE_TRACKER_HOST=resource-usage-tracker + +WEBSERVER_NOTIFICATIONS=1 + +WEBSERVER_CLUSTERS=0 +WEBSERVER_GROUPS=1 +WEBSERVER_META_MODELING=1 +WEBSERVER_PRODUCTS=1 +WEBSERVER_PUBLICATIONS=1 +WEBSERVER_SOCKETIO=1 +WEBSERVER_TAGS=1 +WEBSERVER_USERS=1 +WEBSERVER_VERSION_CONTROL=1 # For development ONLY --------------- # @@ -128,3 +190,70 @@ WEBSERVER_STUDIES_ACCESS_ENABLED=0 # PYTHONTRACEMALLOC=1 # # ------------------------------------ + +AIODEBUG_SLOW_DURATION_SECS=0 + +# Webserver Garbage Collector + +# the same in all deployments +WB_GC_LOGLEVEL=DEBUG + +WB_GC_GARBAGE_COLLECTOR='{"GARBAGE_COLLECTOR_INTERVAL_S": 30}' +WB_GC_RESOURCE_MANAGER_RESOURCE_TTL_S=60 + +WB_GC_ANNOUNCEMENTS=0 +WB_GC_ACTIVITY=null +WB_GC_CATALOG=null +WB_GC_NOTIFICATIONS=0 +WB_GC_DB_LISTENER=0 +WB_GC_DIAGNOSTICS=null +WB_GC_EMAIL=null +WB_GC_EXPORTER=null +WB_GC_FRONTEND=null +WB_GC_LOGIN=null +WB_GC_PROJECTS=null +WB_GC_SCICRUNCH=null +WB_GC_STATICWEB=null + +WB_GC_TRACING=null +WB_GC_CLUSTERS=0 +WB_GC_GROUPS=0 +WB_GC_META_MODELING=0 +WB_GC_PRODUCTS=0 +WB_GC_PUBLICATIONS=0 +WB_GC_SOCKETIO=1 +WB_GC_STUDIES_DISPATCHER=null +WB_GC_TAGS=0 +WB_GC_USERS=0 +WB_GC_VERSION_CONTROL=0 + +# WebServer DB Event Listener + +WB_DB_EL_LOGLEVEL=DEBUG +WB_DB_EL_DB_LISTENER=1 +WB_DB_EL_SOCKETIO=1 + +WB_DB_EL_ANNOUNCEMENTS=0 +WB_DB_EL_ACTIVITY=null +WB_DB_EL_CATALOG=null +WB_DB_EL_NOTIFICATIONS=0 +WB_DB_EL_DIAGNOSTICS=null +WB_DB_EL_EMAIL=null +WB_DB_EL_EXPORTER=null +WB_DB_EL_FRONTEND=null +WB_DB_EL_GARBAGE_COLLECTOR=null +WB_DB_EL_LOGIN=null +WB_DB_EL_PROJECTS=null +WB_DB_EL_SCICRUNCH=null +WB_DB_EL_STATICWEB=null +WB_DB_EL_STORAGE=null +WB_DB_EL_STUDIES_DISPATCHER=null +WB_DB_EL_TRACING=null +WB_DB_EL_CLUSTERS=0 +WB_DB_EL_GROUPS=0 +WB_DB_EL_META_MODELING=0 +WB_DB_EL_PRODUCTS=0 +WB_DB_EL_PUBLICATIONS=0 +WB_DB_EL_TAGS=0 +WB_DB_EL_USERS=0 +WB_DB_EL_VERSION_CONTROL=0 diff --git a/.env-wb-db-event-listener b/.env-wb-db-event-listener deleted file mode 100644 index b8523bd6929..00000000000 --- a/.env-wb-db-event-listener +++ /dev/null @@ -1,38 +0,0 @@ -# -# Explicit plugins DISABLED in the webserver to create a garbage-collector -# Docs plugins config of services/web/server/src/simcore_service_webserver/application_settings.py -# - -WEBSERVER_ANNOUNCEMENTS=0 -WEBSERVER_ACTIVITY=null -WEBSERVER_CATALOG=null -WEBSERVER_NOTIFICATIONS=0 -# WEBSERVER_DB_LISTENER=1 explicitly enabled in docker-compose -WEBSERVER_DIAGNOSTICS=null -#WEBSERVER_DIRECTOR_V2 from .env -WEBSERVER_EMAIL=null -WEBSERVER_EXPORTER=null -WEBSERVER_FRONTEND=null -WEBSERVER_GARBAGE_COLLECTOR=null -WEBSERVER_LOGIN=null -WEBSERVER_PROJECTS=null -#WEBSERVER_RABBITMQ from .env -#WEBSERVER_REDIS from .env -#WEBSERVER_REST needed for the healthcheck -#WEBSERVER_RESOURCE_MANAGER from .env -WEBSERVER_SCICRUNCH=null -WEBSERVER_STATICWEB=null -WEBSERVER_STORAGE=null -WEBSERVER_STUDIES_DISPATCHER=null -WEBSERVER_TRACING=null -# -------- -WEBSERVER_CLUSTERS=0 -WEBSERVER_GROUPS=0 -WEBSERVER_META_MODELING=0 -WEBSERVER_PRODUCTS=0 -WEBSERVER_PUBLICATIONS=0 -#WEBSERVER_SOCKETIO=1 explicitly enabled in docker-compose -WEBSERVER_STUDIES_DISPATCHER=null -WEBSERVER_TAGS=0 -WEBSERVER_USERS=0 -WEBSERVER_VERSION_CONTROL=0 diff --git a/.env-wb-garbage-collector b/.env-wb-garbage-collector deleted file mode 100644 index 96f189916ff..00000000000 --- a/.env-wb-garbage-collector +++ /dev/null @@ -1,38 +0,0 @@ -# -# Explicit plugins DISABLED in the webserver to create a garbage-collector -# Docs plugins config of services/web/server/src/simcore_service_webserver/application_settings.py -# - - -WEBSERVER_ANNOUNCEMENTS=0 -WEBSERVER_ACTIVITY=null -WEBSERVER_CATALOG=null -WEBSERVER_NOTIFICATIONS=0 -WEBSERVER_DB_LISTENER=0 -WEBSERVER_DIAGNOSTICS=null -#WEBSERVER_DIRECTOR_V2 from .env -WEBSERVER_EMAIL=null -WEBSERVER_EXPORTER=null -WEBSERVER_FRONTEND=null -#WEBSERVER_GARBAGE_COLLECTOR explicitely enabled in docker-compose -WEBSERVER_LOGIN=null -WEBSERVER_PROJECTS=null -#WEBSERVER_REDIS from .env -#WEBSERVER_REST needed for the healthcheck -#WEBSERVER_RESOURCE_MANAGER from .env -WEBSERVER_SCICRUNCH=null -WEBSERVER_STATICWEB=null -# WEBSERVER_STORAGE needed to delete data when removing anonymous service -WEBSERVER_STUDIES_DISPATCHER=null -WEBSERVER_TRACING=null -# -------- -WEBSERVER_CLUSTERS=0 -WEBSERVER_GROUPS=0 -WEBSERVER_META_MODELING=0 -WEBSERVER_PRODUCTS=0 -WEBSERVER_PUBLICATIONS=0 -WEBSERVER_SOCKETIO=1 -WEBSERVER_STUDIES_DISPATCHER=null -WEBSERVER_TAGS=0 -WEBSERVER_USERS=0 -WEBSERVER_VERSION_CONTROL=0 diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index 8df034fb255..6f76223a6bd 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -95,7 +95,7 @@ services: volumes: *webserver-volumes-dev environment: <<: *webserver-environment-dev - WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS: 15 + RESOURCE_MANAGER_RESOURCE_TTL_S: 15 dask-sidecar: volumes: @@ -103,6 +103,7 @@ services: - ./dask-sidecar:/devel/services/dask-sidecar - ../packages:/devel/packages - ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro + environment: SC_BOOT_MODE: debug-ptvsd SIDECAR_LOGLEVEL: DEBUG diff --git a/services/docker-compose.yml b/services/docker-compose.yml index aa5e12c85ca..ee320d3634c 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -261,27 +261,146 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" environment: - &webserver-environment - CATALOG_HOST: ${CATALOG_HOST:-catalog} - CATALOG_PORT: ${CATALOG_PORT:-8000} - DIAGNOSTICS_MAX_AVG_LATENCY: 10 - DIAGNOSTICS_MAX_TASK_DELAY: 30 - DIRECTOR_HOST: ${DIRECTOR_HOST:-director} - DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST:-director-v2} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT:-8000} + AIODEBUG_SLOW_DURATION_SECS: ${AIODEBUG_SLOW_DURATION_SECS} + + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + + WEBSERVER_DEV_FEATURES_ENABLED: ${WEBSERVER_DEV_FEATURES_ENABLED} + + WEBSERVER_LOGLEVEL: ${WEBSERVER_LOGLEVEL} + + WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + # WEBSERVER_SERVER_HOST + + WEBSERVER_HOST: ${WEBSERVER_HOST} + + # WEBSERVER_SERVER_PORT + + WEBSERVER_FRONTEND: ${WEBSERVER_FRONTEND} + + # WEBSERVER_ACTIVITY + WEBSERVER_ACTIVITY: ${WEBSERVER_ACTIVITY} + PROMETHEUS_API_VERSION: ${WEBSERVER_PROMETHEUS_API_VERSION} # seems to be not used + PROMETHEUS_URL: ${WEBSERVER_PROMETHEUS_URL} + + WEBSERVER_CATALOG: ${WEBSERVER_CATALOG} + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + + # WEBSERVER_DB + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + + # WEBSERVER_DIAGNOSTICS + WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS} + DIAGNOSTICS_MAX_AVG_LATENCY: ${DIAGNOSTICS_MAX_AVG_LATENCY} + DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY} + DIAGNOSTICS_SLOW_DURATION_SECS: ${DIAGNOSTICS_SLOW_DURATION_SECS} + + # WEBSERVER_DIRECTOR_V2 + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + # WEBSERVER_DIRECTOR + DIRECTOR_HOST: ${DIRECTOR_HOST} + DIRECTOR_PORT: ${DIRECTOR_PORT} + + # WEBSERVER_EMAIL + WEBSERVER_EMAIL: ${WEBSERVER_EMAIL} + SMTP_HOST: ${SMTP_HOST} + SMTP_PORT: ${SMTP_PORT} + + WEBSERVER_EXPORTER: ${WEBSERVER_EXPORTER} + + # WEBSERVER_GARBAGE_COLLECTOR + WEBSERVER_GARBAGE_COLLECTOR: ${WEBSERVER_GARBAGE_COLLECTOR} + + # WEBSERVER_INVITATIONS + INVITATIONS_HOST: ${INVITATIONS_HOST} + INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} + INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} + INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} + INVITATIONS_PORT: ${INVITATIONS_PORT} + INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} + INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} + + WEBSERVER_LOGIN: ${WEBSERVER_LOGIN} + LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: ${LOGIN_REGISTRATION_CONFIRMATION_REQUIRED} + LOGIN_REGISTRATION_INVITATION_REQUIRED: ${LOGIN_REGISTRATION_INVITATION_REQUIRED} + LOGIN_2FA_REQUIRED: ${LOGIN_2FA_REQUIRED} + LOGIN_2FA_CODE_EXPIRATION_SEC: ${LOGIN_2FA_CODE_EXPIRATION_SEC} + + # WEBSERVER_REDIS + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + + # WEBSERVER_REST + + # WEBSERVER_RESOURCE_MANAGER + RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + + # WEBSERVER_SCICRUNCH + WEBSERVER_SCICRUNCH: ${WEBSERVER_SCICRUNCH} + SCICRUNCH_API_BASE_URL: ${SCICRUNCH_API_BASE_URL} + SCICRUNCH_API_KEY: ${SCICRUNCH_API_KEY} + + # WEBSERVER_SESSION + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + + WEBSERVER_STATICWEB: ${WEBSERVER_STATICWEB} + + # WEBSERVER_STORAGE + STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + + # WEBSERVER_STUDIES_DISPATCHER + WEBSERVER_STUDIES_DISPATCHER: ${WEBSERVER_STUDIES_DISPATCHER} + STUDIES_ACCESS_ANONYMOUS_ALLOWED: ${STUDIES_ACCESS_ANONYMOUS_ALLOWED} + STUDIES_DEFAULT_SERVICE_THUMBNAIL: ${STUDIES_DEFAULT_SERVICE_THUMBNAIL} + + WEBSERVER_TRACING: ${WEBSERVER_TRACING} + TRACING_ENABLED: ${TRACING_ENABLED} + TRACING_ZIPKIN_ENDPOINT: ${TRACING_ZIPKIN_ENDPOINT} + TRACING_THRIFT_COMPACT_ENDPOINT: ${TRACING_THRIFT_COMPACT_ENDPOINT} + + # WEBSERVER_PROJECTS + WEBSERVER_PROJECTS: ${WEBSERVER_PROJECTS} + PROJECTS_MAX_COPY_SIZE_BYTES: ${PROJECTS_MAX_COPY_SIZE_BYTES} + PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES: ${PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES} + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_USER: ${RABBIT_USER} + + # ARBITRARY ENV VARS + # see [https://docs.gunicorn.org/en/stable/settings.html#timeout], # since we have the docker healthcheck already, this should be ok - GUNICORN_CMD_ARGS: --timeout=180 - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - STORAGE_HOST: ${STORAGE_HOST:-storage} - STORAGE_PORT: ${STORAGE_PORT:-8080} - SWARM_STACK_NAME: ${SWARM_STACK_NAME:-simcore} - WEBSERVER_DB_LISTENER: 0 - WEBSERVER_GARBAGE_COLLECTOR: "null" - WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-WARNING} - env_file: - - ../.env + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + WEBSERVER_DB_LISTENER: ${WEBSERVER_DB_LISTENER} + WEBSERVER_ANNOUNCEMENTS: ${WEBSERVER_ANNOUNCEMENTS} + WEBSERVER_NOTIFICATIONS: ${WEBSERVER_NOTIFICATIONS} + WEBSERVER_CLUSTERS: ${WEBSERVER_CLUSTERS} + WEBSERVER_GROUPS: ${WEBSERVER_GROUPS} + WEBSERVER_META_MODELING: ${WEBSERVER_META_MODELING} + WEBSERVER_PRODUCTS: ${WEBSERVER_PRODUCTS} + WEBSERVER_PUBLICATIONS: ${WEBSERVER_PUBLICATIONS} + WEBSERVER_SOCKETIO: ${WEBSERVER_SOCKETIO} + WEBSERVER_TAGS: ${WEBSERVER_TAGS} + WEBSERVER_USERS: ${WEBSERVER_USERS} + WEBSERVER_VERSION_CONTROL: ${WEBSERVER_VERSION_CONTROL} + deploy: placement: constraints: @@ -312,17 +431,87 @@ services: networks: - default - interactive_services_subnet + wb-db-event-listener: image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} init: true hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" environment: - <<: *webserver-environment - WEBSERVER_DB_LISTENER: 1 - WEBSERVER_SOCKETIO: 1 - env_file: - - ../.env - - ../.env-wb-db-event-listener + WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} + + WEBSERVER_HOST: ${WEBSERVER_HOST} + + # WEBSERVER_DB + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + + DIRECTOR_HOST: ${DIRECTOR_HOST} + DIRECTOR_PORT: ${DIRECTOR_PORT} + + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + REST_SWAGGER_API_DOC_ENABLED: ${REST_SWAGGER_API_DOC_ENABLED} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + + # WEBSERVER_INVITATIONS + INVITATIONS_HOST: ${INVITATIONS_HOST} + INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} + INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} + INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} + INVITATIONS_PORT: ${INVITATIONS_PORT} + INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} + INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} + + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} + WEBSERVER_SOCKETIO: ${WB_DB_EL_SOCKETIO} + WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} + WEBSERVER_ACTIVITY: ${WB_DB_EL_ACTIVITY} + WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} + WEBSERVER_NOTIFICATIONS: ${WB_DB_EL_NOTIFICATIONS} + WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} + WEBSERVER_EMAIL: ${WB_DB_EL_EMAIL} + WEBSERVER_EXPORTER: ${WB_DB_EL_EXPORTER} + WEBSERVER_FRONTEND: ${WB_DB_EL_FRONTEND} + WEBSERVER_GARBAGE_COLLECTOR: ${WB_DB_EL_GARBAGE_COLLECTOR} + WEBSERVER_LOGIN: ${WB_DB_EL_LOGIN} + WEBSERVER_PROJECTS: ${WB_DB_EL_PROJECTS} + WEBSERVER_SCICRUNCH: ${WB_DB_EL_SCICRUNCH} + WEBSERVER_STATICWEB: ${WB_DB_EL_STATICWEB} + WEBSERVER_STORAGE: ${WB_DB_EL_STORAGE} + WEBSERVER_STUDIES_DISPATCHER: ${WB_DB_EL_STUDIES_DISPATCHER} + WEBSERVER_TRACING: ${WB_DB_EL_TRACING} + WEBSERVER_CLUSTERS: ${WB_DB_EL_CLUSTERS} + WEBSERVER_GROUPS: ${WB_DB_EL_GROUPS} + WEBSERVER_META_MODELING: ${WB_DB_EL_META_MODELING} + WEBSERVER_PRODUCTS: ${WB_DB_EL_PRODUCTS} + WEBSERVER_PUBLICATIONS: ${WB_DB_EL_PUBLICATIONS} + WEBSERVER_TAGS: ${WB_DB_EL_TAGS} + WEBSERVER_USERS: ${WB_DB_EL_USERS} + WEBSERVER_VERSION_CONTROL: ${WB_DB_EL_VERSION_CONTROL} + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_USER: ${RABBIT_USER} + + # WEBSERVER_REDIS + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + + RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} + deploy: # NOTE: https://github.com/ITISFoundation/osparc-simcore/pull/4286 # NOTE: this MUSTN'T change, or weird things might happen @@ -339,12 +528,86 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" environment: - <<: *webserver-environment - WEBSERVER_GARBAGE_COLLECTOR: '{"GARBAGE_COLLECTOR_INTERVAL_S": 30}' - WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS: 60 - env_file: - - ../.env - - ../.env-wb-garbage-collector + WEBSERVER_LOGLEVEL: ${WB_GC_LOGLEVEL} + + WEBSERVER_HOST: ${WEBSERVER_HOST} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + + REST_SWAGGER_API_DOC_ENABLED: ${REST_SWAGGER_API_DOC_ENABLED} + + INVITATIONS_HOST: ${INVITATIONS_HOST} + INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} + INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} + INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} + INVITATIONS_PORT: ${INVITATIONS_PORT} + INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} + INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} + + # WEBSERVER_DB + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + + DIRECTOR_HOST: ${DIRECTOR_HOST} + DIRECTOR_PORT: ${DIRECTOR_PORT} + + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + + WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} + + WEBSERVER_GARBAGE_COLLECTOR: ${WB_GC_GARBAGE_COLLECTOR} + + RESOURCE_MANAGER_RESOURCE_TTL_S: ${WB_GC_RESOURCE_MANAGER_RESOURCE_TTL_S} + + WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} + WEBSERVER_ACTIVITY: ${WB_GC_ACTIVITY} + WEBSERVER_CATALOG: ${WB_GC_CATALOG} + WEBSERVER_NOTIFICATIONS: ${WB_GC_NOTIFICATIONS} + WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} + WEBSERVER_EMAIL: ${WB_GC_EMAIL} + WEBSERVER_EXPORTER: ${WB_GC_EXPORTER} + WEBSERVER_FRONTEND: ${WB_GC_FRONTEND} + WEBSERVER_LOGIN: ${WB_GC_LOGIN} + WEBSERVER_PROJECTS: ${WB_GC_PROJECTS} + WEBSERVER_SCICRUNCH: ${WB_GC_SCICRUNCH} + WEBSERVER_STATICWEB: ${WB_GC_STATICWEB} + WEBSERVER_STUDIES_DISPATCHER: ${WB_GC_STUDIES_DISPATCHER} + WEBSERVER_TRACING: ${WB_GC_TRACING} + WEBSERVER_CLUSTERS: ${WB_GC_CLUSTERS} + WEBSERVER_GROUPS: ${WB_GC_GROUPS} + WEBSERVER_META_MODELING: ${WB_GC_META_MODELING} + WEBSERVER_PRODUCTS: ${WB_GC_PRODUCTS} + WEBSERVER_PUBLICATIONS: ${WB_GC_PUBLICATIONS} + WEBSERVER_SOCKETIO: ${WB_GC_SOCKETIO} + WEBSERVER_TAGS: ${WB_GC_TAGS} + WEBSERVER_USERS: ${WB_GC_USERS} + WEBSERVER_VERSION_CONTROL: ${WB_GC_VERSION_CONTROL} + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_USER: ${RABBIT_USER} + deploy: placement: constraints: diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index b56c3cd6a10..782f5933b37 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -206,9 +206,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_ANNOUNCEMENTS: bool = False WEBSERVER_CLUSTERS: bool = False WEBSERVER_DB_LISTENER: bool = True - WEBSERVER_NOTIFICATIONS: bool = Field( - default=True, env=["WEBSERVER_NOTIFICATIONS", "WEBSERVER_COMPUTATION"] - ) + WEBSERVER_NOTIFICATIONS: bool = Field(default=True) WEBSERVER_GROUPS: bool = True WEBSERVER_META_MODELING: bool = True WEBSERVER_PRODUCTS: bool = True diff --git a/services/web/server/src/simcore_service_webserver/application_settings_utils.py b/services/web/server/src/simcore_service_webserver/application_settings_utils.py index c1a1c0f2673..57787c5ed61 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings_utils.py +++ b/services/web/server/src/simcore_service_webserver/application_settings_utils.py @@ -215,10 +215,10 @@ def _set_if_disabled(field_name, section): if section := cfg.get("resource_manager"): _set_if_disabled("WEBSERVER_RESOURCE_MANAGER", section) - envs["WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS"] = section.get( + envs["RESOURCE_MANAGER_RESOURCE_TTL_S"] = section.get( "resource_deletion_timeout_seconds" ) - envs["WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS"] = section.get( + envs["GARBAGE_COLLECTOR_INTERVAL_S"] = section.get( "garbage_collection_interval_seconds" ) diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/settings.py b/services/web/server/src/simcore_service_webserver/garbage_collector/settings.py index addac999df1..46863d45864 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/settings.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/settings.py @@ -16,11 +16,6 @@ class GarbageCollectorSettings(BaseCustomSettings): GARBAGE_COLLECTOR_INTERVAL_S: PositiveInt = Field( 30 * _SEC, description="Waiting time between consecutive runs of the garbage-colector", - # legacy - env=[ - "GARBAGE_COLLECTOR_INTERVAL_S", - "WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS", # legacy - ], ) GARBAGE_COLLECTOR_EXPIRED_USERS_CHECK_INTERVAL_S: PositiveInt = Field( diff --git a/services/web/server/src/simcore_service_webserver/login/settings.py b/services/web/server/src/simcore_service_webserver/login/settings.py index cc6e44eb586..fb3d8671c35 100644 --- a/services/web/server/src/simcore_service_webserver/login/settings.py +++ b/services/web/server/src/simcore_service_webserver/login/settings.py @@ -23,18 +23,10 @@ class LoginSettings(BaseCustomSettings): LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: bool = Field( True, - env=[ - "LOGIN_REGISTRATION_CONFIRMATION_REQUIRED", - "WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED", - ], ) LOGIN_REGISTRATION_INVITATION_REQUIRED: bool = Field( ..., - env=[ - "LOGIN_REGISTRATION_INVITATION_REQUIRED", - "WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED", - ], ) LOGIN_TWILIO: TwilioSettings | None = Field( diff --git a/services/web/server/src/simcore_service_webserver/resource_manager/settings.py b/services/web/server/src/simcore_service_webserver/resource_manager/settings.py index 1d40c52f10b..b3b954667ad 100644 --- a/services/web/server/src/simcore_service_webserver/resource_manager/settings.py +++ b/services/web/server/src/simcore_service_webserver/resource_manager/settings.py @@ -9,11 +9,6 @@ class ResourceManagerSettings(BaseCustomSettings): RESOURCE_MANAGER_RESOURCE_TTL_S: PositiveInt = Field( 900, description="Expiration time (or Time to live (TTL) in redis jargon) for a registered resource", - # legacy! - env=[ - "RESOURCE_MANAGER_RESOURCE_TTL_S", - "WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS", # legacy - ], ) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py index a32a5d63a95..d831ec37d02 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py @@ -11,7 +11,6 @@ class StudiesDispatcherSettings(BaseCustomSettings): STUDIES_ACCESS_ANONYMOUS_ALLOWED: bool = Field( False, description="If enabled, the study links are accessible to anonymous users", - env=["STUDIES_ACCESS_ANONYMOUS_ALLOWED", "WEBSERVER_STUDIES_ACCESS_ENABLED"], ) STUDIES_GUEST_ACCOUNT_LIFETIME: timedelta = Field( diff --git a/services/web/server/tests/data/default_app_config-integration.yaml b/services/web/server/tests/data/default_app_config-integration.yaml index 66b8cc31c6e..364cf3430e2 100644 --- a/services/web/server/tests/data/default_app_config-integration.yaml +++ b/services/web/server/tests/data/default_app_config-integration.yaml @@ -9,8 +9,8 @@ main: testing: true activity: enabled: true - prometheus_api_version: ${WEBSERVER_PROMETHEUS_API_VERSION} - prometheus_url: ${WEBSERVER_PROMETHEUS_URL} + prometheus_api_version: ${PROMETHEUS_API_VERSION} + prometheus_url: ${PROMETHEUS_URL} catalog: enabled: true host: ${CATALOG_HOST} @@ -60,17 +60,17 @@ remote_debug: enabled: false resource_manager: enabled: true - garbage_collection_interval_seconds: ${WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS} + garbage_collection_interval_seconds: ${GARBAGE_COLLECTOR_INTERVAL_S} redis: host: ${REDIS_HOST} port: ${REDIS_PORT} - resource_deletion_timeout_seconds: ${WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS} + resource_deletion_timeout_seconds: ${RESOURCE_MANAGER_RESOURCE_TTL_S} rest: enabled: true version: v0 session: enabled: true - secret_key: ${WEBSERVER_SESSION_SECRET_KEY} + secret_key: ${SESSION_SECRET_KEY} security: enabled: true smtp: diff --git a/services/web/server/tests/integration/conftest.py b/services/web/server/tests/integration/conftest.py index f2f9937a8e0..6154c99b89a 100644 --- a/services/web/server/tests/integration/conftest.py +++ b/services/web/server/tests/integration/conftest.py @@ -72,10 +72,12 @@ def webserver_environ( # version tha loads only the subsystems under test. For that reason, # the test webserver is built-up in webserver_service fixture that runs # on the host. + EXCLUDED_SERVICES = ["dask-scheduler"] services_with_published_ports = [ name for name in core_services if "ports" in simcore_docker_compose["services"][name] + and name not in EXCLUDED_SERVICES ] for name in services_with_published_ports: host_key = f"{name.upper().replace('-', '_')}_HOST" @@ -135,7 +137,10 @@ def _default_app_config_for_integration_tests( # for the moment using web-server as an all-in-one service. # TODO: create integration tests using different configs # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2896 - test_environ["WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS"] = "30" + test_environ[ + "WEBSERVER_GARBAGE_COLLECTOR" + ] = "{}" # by default it is disabled. This enables it with default or env variables + test_environ["GARBAGE_COLLECTOR_INTERVAL_S"] = "30" # recreate config-file config_template = Template(default_app_config_integration_file.read_text()) diff --git a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py index afd8b263284..0681e5dce21 100644 --- a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py +++ b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py @@ -77,7 +77,7 @@ def mock_environment(mock_env_devel_environment: dict[str, str], monkeypatch): monkeypatch.setenv("AIODEBUG_SLOW_DURATION_SECS", f"{SLOW_HANDLER_DELAY_SECS / 10}") monkeypatch.setenv("DIAGNOSTICS_MAX_TASK_DELAY", f"{SLOW_HANDLER_DELAY_SECS}") monkeypatch.setenv("DIAGNOSTICS_MAX_AVG_LATENCY", f"{2.0}") - monkeypatch.setenv("DIAGNOSTICS_START_SENSING_DELAY", f"{0}") # inmidiately + monkeypatch.setenv("DIAGNOSTICS_START_SENSING_DELAY", f"{0}") monkeypatch.setenv("SC_HEALTHCHECK_TIMEOUT", "2m") diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py index a8ec3e68165..959f6def3c2 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py @@ -41,7 +41,7 @@ def app_environment( "WEBSERVER_ACTIVITY": "null", "WEBSERVER_CATALOG": "null", "WEBSERVER_CLUSTERS": "false", - "WEBSERVER_COMPUTATION": "0", + "WEBSERVER_NOTIFICATIONS": "0", "WEBSERVER_DIAGNOSTICS": "null", "WEBSERVER_DIRECTOR_V2": "null", "WEBSERVER_EXPORTER": "null", diff --git a/services/web/server/tests/unit/with_dbs/03/login/conftest.py b/services/web/server/tests/unit/with_dbs/03/login/conftest.py index bb446044c32..4bd102bfd05 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/login/conftest.py @@ -32,7 +32,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: MonkeyPatch): "WEBSERVER_PUBLICATIONS": "0", "WEBSERVER_REMOTE_DEBUG": "0", "WEBSERVER_SOCKETIO": "1", # for login notifications - "WEBSERVER_STUDIES_ACCESS_ENABLED": "0", + "WEBSERVER_STUDIES_DISPATCHER": "null", "WEBSERVER_TAGS": "1", "WEBSERVER_TRACING": "null", "WEBSERVER_USERS": "1", diff --git a/services/web/server/tests/unit/with_dbs/03/test_email.py b/services/web/server/tests/unit/with_dbs/03/test_email.py index ff44b616306..31d1973b93e 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_email.py +++ b/services/web/server/tests/unit/with_dbs/03/test_email.py @@ -44,7 +44,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: MonkeyPatch): "WEBSERVER_PUBLICATIONS": "0", "WEBSERVER_REMOTE_DEBUG": "0", "WEBSERVER_SOCKETIO": "1", # for login notifications - "WEBSERVER_STUDIES_ACCESS_ENABLED": "0", + "WEBSERVER_STUDIES_DISPATCHER": "null", "WEBSERVER_TAGS": "1", "WEBSERVER_TRACING": "null", "WEBSERVER_USERS": "1", diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index f16c37f4e6b..109d5221e0b 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -12,8 +12,8 @@ endef define _up_simcore # set some parameters to allow for e2e to run -echo WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV) -echo WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV) +echo LOGIN_REGISTRATION_INVITATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV) +echo LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV) echo DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV) echo DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV) echo SIDECAR_FORCE_CPU_NODE=1 >> $(SIMCORE_DOT_ENV)