♻️ Clean up simcore webserver environment ⚠️ DEVOPS

.env-devel

@@ -81,8 +81,6 @@ S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
 
-
-
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
 
@@ -105,19 +103,80 @@ TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 
 # NOTE: WEBSERVER_SESSION_SECRET_KEY = $(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())")
+
+WEBSERVER_LOGLEVEL=INFO
+
+WEBSERVER_PROJECTS={}
 PROJECTS_MAX_COPY_SIZE_BYTES=30Gib
 PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES=5
+
 WEBSERVER_ANNOUNCEMENTS=1
+
 WEBSERVER_DEV_FEATURES_ENABLED=0
+
 WEBSERVER_HOST=webserver
-WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0
-WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED=0
+
+WEBSERVER_CATALOG={}
+CATALOG_HOST=catalog
+CATALOG_PORT=8000
+
+WEBSERVER_SCICRUNCH={}
+
+WEBSERVER_SESSION_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key='
+
+WEBSERVER_STATICWEB={}
+
+WEBSERVER_STUDIES_DISPATCHER={}
+STUDIES_ACCESS_ANONYMOUS_ALLOWED=0
+STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png
+
+WEBSERVER_TRACING={}
+
+WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180
+
+WEBSERVER_EXPORTER={}
+
+WEBSERVER_EMAIL={}
+
+DIRECTOR_HOST=director
+DIRECTOR_PORT=8080
+
+DIRECTOR_V2_HOST=director-v2
+DIRECTOR_V2_PORT=8000
+
+WEBSERVER_DIAGNOSTICS={}
+DIAGNOSTICS_MAX_AVG_LATENCY=10
+DIAGNOSTICS_MAX_TASK_DELAY=30
+
+WEBSERVER_GARBAGE_COLLECTOR=null
+
+WEBSERVER_DB_LISTENER=0
+
+WEBSERVER_LOGIN={}
+LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0
+LOGIN_REGISTRATION_INVITATION_REQUIRED=0
+LOGIN_2FA_REQUIRED=0
+LOGIN_2FA_CODE_EXPIRATION_SEC=60
+
+WEBSERVER_FRONTEND={}
+
+WEBSERVER_ACTIVITY=null  # disable Prometheus
 WEBSERVER_PROMETHEUS_API_VERSION=v1
 WEBSERVER_PROMETHEUS_URL=http://prometheus:9090
-WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=900
-WEBSERVER_SESSION_SECRET_KEY='REPLACE_ME_with_result__Fernet_generate_key='
-WEBSERVER_STUDIES_ACCESS_ENABLED=0
 
+RESOURCE_MANAGER_RESOURCE_TTL_S=900
+
+WEBSERVER_NOTIFICATIONS=1
+
+WEBSERVER_CLUSTERS=0
+WEBSERVER_GROUPS=1
+WEBSERVER_META_MODELING=1
+WEBSERVER_PRODUCTS=1
+WEBSERVER_PUBLICATIONS=1
+WEBSERVER_SOCKETIO=1
+WEBSERVER_TAGS=1
+WEBSERVER_USERS=1
+WEBSERVER_VERSION_CONTROL=1
 
 # For development ONLY ---------------
 #

services/docker-compose.devel.yml

@@ -95,14 +95,15 @@ services:
     volumes: *webserver-volumes-dev
     environment:
       <<: *webserver-environment-dev
-      WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS: 15
+      RESOURCE_MANAGER_RESOURCE_TTL_S: 15
 
   dask-sidecar:
     volumes:
       &dev-dask-sidecar-volumes
       - ./dask-sidecar:/devel/services/dask-sidecar
       - ../packages:/devel/packages
       - ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro
+
     environment:
       SC_BOOT_MODE: debug-ptvsd
       SIDECAR_LOGLEVEL: DEBUG

services/docker-compose.yml

@@ -262,26 +262,143 @@ services:
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     environment:
       &webserver-environment
-      CATALOG_HOST: ${CATALOG_HOST:-catalog}
-      CATALOG_PORT: ${CATALOG_PORT:-8000}
-      DIAGNOSTICS_MAX_AVG_LATENCY: 10
-      DIAGNOSTICS_MAX_TASK_DELAY: 30
-      DIRECTOR_HOST: ${DIRECTOR_HOST:-director}
-      DIRECTOR_PORT: ${DIRECTOR_PORT:-8080}
-      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST:-director-v2}
-      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT:-8000}
+
+      SWARM_STACK_NAME: ${SWARM_STACK_NAME}
+
+      WEBSERVER_DEV_FEATURES_ENABLED: ${WEBSERVER_DEV_FEATURES_ENABLED}
+
+      WEBSERVER_LOGLEVEL: ${WEBSERVER_LOGLEVEL}
+
+      WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
+
+      # WEBSERVER_SERVER_HOST
+
+      WEBSERVER_HOST: ${WEBSERVER_HOST}
+
+      # WEBSERVER_SERVER_PORT
+
+      WEBSERVER_FRONTEND: ${WEBSERVER_FRONTEND}
+
+      # WEBSERVER_ACTIVITY
+      WEBSERVER_ACTIVITY: ${WEBSERVER_ACTIVITY}
+      PROMETHEUS_API_VERSION: ${WEBSERVER_PROMETHEUS_API_VERSION}  # seems to be not used
+      PROMETHEUS_URL: ${WEBSERVER_PROMETHEUS_URL}
+
+      WEBSERVER_CATALOG: ${WEBSERVER_CATALOG}
+      CATALOG_HOST: ${CATALOG_HOST}
+      CATALOG_PORT: ${CATALOG_PORT}
+
+      # WEBSERVER_DB
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT}
+      POSTGRES_HOST: ${POSTGRES_HOST}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_PORT: ${POSTGRES_PORT}
+      POSTGRES_USER: ${POSTGRES_USER}
+
+      # WEBSERVER_DIAGNOSTICS
+      WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS}
+      DIAGNOSTICS_MAX_AVG_LATENCY: ${DIAGNOSTICS_MAX_AVG_LATENCY}
+      DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY}
+
+      # WEBSERVER_DIRECTOR_V2
+      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
+      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
+
+      # WEBSERVER_DIRECTOR
+      DIRECTOR_HOST: ${DIRECTOR_HOST}
+      DIRECTOR_PORT: ${DIRECTOR_PORT}
+
+      # WEBSERVER_EMAIL
+      WEBSERVER_EMAIL: ${WEBSERVER_EMAIL}
+      SMTP_HOST: ${SMTP_HOST}
+      SMTP_PORT: ${SMTP_PORT}
+
+      WEBSERVER_EXPORTER: ${WEBSERVER_EXPORTER}
+
+      # WEBSERVER_GARBAGE_COLLECTOR
+      WEBSERVER_GARBAGE_COLLECTOR: ${WEBSERVER_GARBAGE_COLLECTOR}
+
+      # WEBSERVER_INVITATIONS
+      INVITATIONS_HOST: ${INVITATIONS_HOST}
+      INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL}
+      INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL}
+      INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD}
+      INVITATIONS_PORT: ${INVITATIONS_PORT}
+      INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY}
+      INVITATIONS_USERNAME: ${INVITATIONS_USERNAME}
+
+      WEBSERVER_LOGIN: ${WEBSERVER_LOGIN}
+      LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: ${LOGIN_REGISTRATION_CONFIRMATION_REQUIRED}
+      LOGIN_REGISTRATION_INVITATION_REQUIRED: ${LOGIN_REGISTRATION_INVITATION_REQUIRED}
+      LOGIN_2FA_REQUIRED: ${LOGIN_2FA_REQUIRED}
+      LOGIN_2FA_CODE_EXPIRATION_SEC: ${LOGIN_2FA_CODE_EXPIRATION_SEC}
+
+      # WEBSERVER_REDIS
+      REDIS_HOST: ${REDIS_HOST}
+      REDIS_PORT: ${REDIS_PORT}
+
+      # WEBSERVER_REST
+
+      # WEBSERVER_RESOURCE_MANAGER
+      RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S}
+
+      # WEBSERVER_RESOURCE_USAGE_TRACKER
+
+      # WEBSERVER_SCICRUNCH
+      WEBSERVER_SCICRUNCH: ${WEBSERVER_SCICRUNCH}
+      SCICRUNCH_API_BASE_URL: ${SCICRUNCH_API_BASE_URL}
+      SCICRUNCH_API_KEY: ${SCICRUNCH_API_KEY}
+
+      # WEBSERVER_SESSION
+      SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY}
+
+      WEBSERVER_STATICWEB: ${WEBSERVER_STATICWEB}
+
+      # WEBSERVER_STORAGE
+      STORAGE_ENDPOINT: ${STORAGE_ENDPOINT}
+      STORAGE_HOST: ${STORAGE_HOST}
+      STORAGE_PORT: ${STORAGE_PORT}
+
+      # WEBSERVER_STUDIES_DISPATCHER
+      WEBSERVER_STUDIES_DISPATCHER: ${WEBSERVER_STUDIES_DISPATCHER}
+      STUDIES_ACCESS_ANONYMOUS_ALLOWED: ${STUDIES_ACCESS_ANONYMOUS_ALLOWED}
+      STUDIES_DEFAULT_SERVICE_THUMBNAIL: ${STUDIES_DEFAULT_SERVICE_THUMBNAIL}
+
+      WEBSERVER_TRACING: ${WEBSERVER_TRACING}
+      TRACING_ENABLED: ${TRACING_ENABLED}
+      TRACING_ZIPKIN_ENDPOINT: ${TRACING_ZIPKIN_ENDPOINT}
+      TRACING_THRIFT_COMPACT_ENDPOINT: ${TRACING_THRIFT_COMPACT_ENDPOINT}
+
+      # WEBSERVER_PROJECTS
+      WEBSERVER_PROJECTS: ${WEBSERVER_PROJECTS}
+      PROJECTS_MAX_COPY_SIZE_BYTES: ${PROJECTS_MAX_COPY_SIZE_BYTES}
+      PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES: ${PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES}
+
+      # WEBSERVER_RABBITMQ
+      RABBIT_HOST: ${RABBIT_HOST}
+      RABBIT_PASSWORD: ${RABBIT_PASSWORD}
+      RABBIT_PORT: ${RABBIT_PORT}
+      RABBIT_USER: ${RABBIT_USER}
+
+      # ARBITRARY ENV VARS
+
       # see [https://docs.gunicorn.org/en/stable/settings.html#timeout],
       # since we have the docker healthcheck already, this should be ok
-      GUNICORN_CMD_ARGS: --timeout=180
-      LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
-      STORAGE_HOST: ${STORAGE_HOST:-storage}
-      STORAGE_PORT: ${STORAGE_PORT:-8080}
-      SWARM_STACK_NAME: ${SWARM_STACK_NAME:-simcore}
-      WEBSERVER_DB_LISTENER: 0
-      WEBSERVER_GARBAGE_COLLECTOR: "null"
-      WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-WARNING}
-    env_file:
-      - ../.env
+      GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS}
+      WEBSERVER_DB_LISTENER: ${WEBSERVER_DB_LISTENER}
+      WEBSERVER_ANNOUNCEMENTS: ${WEBSERVER_ANNOUNCEMENTS}
+      WEBSERVER_NOTIFICATIONS: ${WEBSERVER_NOTIFICATIONS}
+      WEBSERVER_CLUSTERS: ${WEBSERVER_CLUSTERS}
+      WEBSERVER_GROUPS: ${WEBSERVER_GROUPS}
+      WEBSERVER_META_MODELING: ${WEBSERVER_META_MODELING}
+      WEBSERVER_PRODUCTS: ${WEBSERVER_PRODUCTS}
+      WEBSERVER_PUBLICATIONS: ${WEBSERVER_PUBLICATIONS}
+      WEBSERVER_SOCKETIO: ${WEBSERVER_SOCKETIO}
+      WEBSERVER_TAGS: ${WEBSERVER_TAGS}
+      WEBSERVER_USERS: ${WEBSERVER_USERS}
+      WEBSERVER_VERSION_CONTROL: ${WEBSERVER_VERSION_CONTROL}
+
     deploy:
       placement:
         constraints:
@@ -312,12 +429,22 @@ services:
     networks:
       - default
       - interactive_services_subnet
+
   wb-db-event-listener:
     image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest}
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     environment:
-      <<: *webserver-environment
+      DIRECTOR_HOST: ${DIRECTOR_HOST}
+      DIRECTOR_PORT: ${DIRECTOR_PORT}
+      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
+      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
+      # see [https://docs.gunicorn.org/en/stable/settings.html#timeout],
+      # since we have the docker healthcheck already, this should be ok
+      GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS}
+      LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
+      SWARM_STACK_NAME: ${SWARM_STACK_NAME}
+      WEBSERVER_LOGLEVEL: ${LOG_LEVEL}
       WEBSERVER_DB_LISTENER: 1
       WEBSERVER_SOCKETIO: 1
     env_file:
@@ -339,9 +466,21 @@ services:
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     environment:
-      <<: *webserver-environment
+      DIRECTOR_HOST: ${DIRECTOR_HOST}
+      DIRECTOR_PORT: ${DIRECTOR_PORT}
+      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
+      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
+      # see [https://docs.gunicorn.org/en/stable/settings.html#timeout],
+      # since we have the docker healthcheck already, this should be ok
+      GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS}
+      LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
+      STORAGE_HOST: ${STORAGE_HOST}
+      STORAGE_PORT: ${STORAGE_PORT}
+      SWARM_STACK_NAME: ${SWARM_STACK_NAME}
+      WEBSERVER_DB_LISTENER: 0
+      WEBSERVER_LOGLEVEL: ${LOG_LEVEL}
       WEBSERVER_GARBAGE_COLLECTOR: '{"GARBAGE_COLLECTOR_INTERVAL_S": 30}'
-      WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS: 60
+      RESOURCE_MANAGER_RESOURCE_TTL_S: 60
     env_file:
       - ../.env
       - ../.env-wb-garbage-collector

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -206,9 +206,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
     WEBSERVER_ANNOUNCEMENTS: bool = False
     WEBSERVER_CLUSTERS: bool = False
     WEBSERVER_DB_LISTENER: bool = True
-    WEBSERVER_NOTIFICATIONS: bool = Field(
-        default=True, env=["WEBSERVER_NOTIFICATIONS", "WEBSERVER_COMPUTATION"]
-    )
+    WEBSERVER_NOTIFICATIONS: bool = Field(default=True)
     WEBSERVER_GROUPS: bool = True
     WEBSERVER_META_MODELING: bool = True
     WEBSERVER_PRODUCTS: bool = True

services/web/server/src/simcore_service_webserver/application_settings_utils.py

@@ -215,10 +215,10 @@ def _set_if_disabled(field_name, section):
     if section := cfg.get("resource_manager"):
         _set_if_disabled("WEBSERVER_RESOURCE_MANAGER", section)
 
-        envs["WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS"] = section.get(
+        envs["RESOURCE_MANAGER_RESOURCE_TTL_S"] = section.get(
             "resource_deletion_timeout_seconds"
         )
-        envs["WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS"] = section.get(
+        envs["GARBAGE_COLLECTOR_INTERVAL_S"] = section.get(
             "garbage_collection_interval_seconds"
         )
 

services/web/server/src/simcore_service_webserver/garbage_collector_settings.py

@@ -16,11 +16,6 @@ class GarbageCollectorSettings(BaseCustomSettings):
     GARBAGE_COLLECTOR_INTERVAL_S: PositiveInt = Field(
         30 * _SEC,
         description="Waiting time between consecutive runs of the garbage-colector",
-        # legacy
-        env=[
-            "GARBAGE_COLLECTOR_INTERVAL_S",
-            "WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS",  # legacy
-        ],
     )
 
     GARBAGE_COLLECTOR_EXPIRED_USERS_CHECK_INTERVAL_S: PositiveInt = Field(

services/web/server/src/simcore_service_webserver/login/settings.py

@@ -23,18 +23,10 @@
 class LoginSettings(BaseCustomSettings):
     LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: bool = Field(
         True,
-        env=[
-            "LOGIN_REGISTRATION_CONFIRMATION_REQUIRED",
-            "WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED",
-        ],
     )
 
     LOGIN_REGISTRATION_INVITATION_REQUIRED: bool = Field(
         ...,
-        env=[
-            "LOGIN_REGISTRATION_INVITATION_REQUIRED",
-            "WEBSERVER_LOGIN_REGISTRATION_INVITATION_REQUIRED",
-        ],
     )
 
     LOGIN_TWILIO: TwilioSettings | None = Field(

services/web/server/src/simcore_service_webserver/resource_manager/settings.py

@@ -9,11 +9,6 @@ class ResourceManagerSettings(BaseCustomSettings):
     RESOURCE_MANAGER_RESOURCE_TTL_S: PositiveInt = Field(
         900,
         description="Expiration time (or Time to live (TTL) in redis jargon) for a registered resource",
-        # legacy!
-        env=[
-            "RESOURCE_MANAGER_RESOURCE_TTL_S",
-            "WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS",  # legacy
-        ],
     )
 
 

services/web/server/src/simcore_service_webserver/studies_dispatcher/settings.py

@@ -11,7 +11,6 @@ class StudiesDispatcherSettings(BaseCustomSettings):
     STUDIES_ACCESS_ANONYMOUS_ALLOWED: bool = Field(
         False,
         description="If enabled, the study links are accessible to anonymous users",
-        env=["STUDIES_ACCESS_ANONYMOUS_ALLOWED", "WEBSERVER_STUDIES_ACCESS_ENABLED"],
     )
 
     STUDIES_GUEST_ACCOUNT_LIFETIME: timedelta = Field(