From 357a84110232a4a63eb38f72ab7d21fe26e27ede Mon Sep 17 00:00:00 2001 From: Matus Drobuliak <60785969+matusdrobuliak66@users.noreply.github.com> Date: Fri, 1 Nov 2024 09:47:57 +0100 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8E=A8=20introduce=20logger=20filteri?= =?UTF-8?q?ng=20with=20ENV=20var=20(#6596)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env-devel | 29 +++--- .../src/servicelib/logging_utils.py | 24 ++++- .../src/servicelib/logging_utils_filtering.py | 28 ++++++ .../tests/test_logging_utils_filtering.py | 99 +++++++++++++++++++ .../simcore_service_agent/core/application.py | 3 +- .../simcore_service_agent/core/settings.py | 6 ++ .../core/application.py | 3 +- .../core/settings.py | 6 ++ .../core/settings.py | 6 ++ .../src/simcore_service_autoscaling/main.py | 3 +- .../simcore_service_catalog/core/settings.py | 6 ++ .../src/simcore_service_catalog/main.py | 3 +- .../core/settings.py | 8 ++ .../simcore_service_clusters_keeper/main.py | 3 +- .../simcore_service_dask_sidecar/settings.py | 6 ++ .../src/simcore_service_dask_sidecar/tasks.py | 3 +- .../core/application.py | 3 +- .../core/settings.py | 8 ++ .../core/application.py | 3 +- .../core/settings.py | 6 ++ services/docker-compose.yml | 17 ++++ .../core/settings.py | 8 ++ .../simcore_service_dynamic_scheduler/main.py | 3 +- .../core/application.py | 3 +- .../core/settings.py | 6 ++ .../core/settings.py | 6 ++ .../src/simcore_service_efs_guardian/main.py | 3 +- .../core/settings.py | 6 ++ .../src/simcore_service_invitations/main.py | 3 +- .../simcore_service_payments/core/settings.py | 6 ++ .../src/simcore_service_payments/main.py | 3 +- .../core/settings.py | 8 ++ .../web_main.py | 3 +- .../src/simcore_service_storage/cli.py | 3 +- .../src/simcore_service_storage/settings.py | 6 ++ .../application_settings.py | 6 ++ .../src/simcore_service_webserver/cli.py | 1 + .../src/simcore_service_webserver/log.py | 8 +- .../01/studies_dispatcher/conftest.py | 4 +- .../with_dbs/03/meta_modeling/conftest.py | 4 +- .../with_dbs/03/version_control/conftest.py | 4 +- 41 files changed, 330 insertions(+), 37 deletions(-) create mode 100644 packages/service-library/src/servicelib/logging_utils_filtering.py create mode 100644 packages/service-library/tests/test_logging_utils_filtering.py diff --git a/.env-devel b/.env-devel index 17aba60a7dd..02f2a9d939e 100644 --- a/.env-devel +++ b/.env-devel @@ -10,7 +10,7 @@ # unset $(grep -v '^#' .env | sed -E 's/(.*)=.*/\1/' | xargs) # -AGENT_LOGLEVEL=WARNING +AGENT_LOGLEVEL=INFO AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY=12345678 AGENT_VOLUMES_CLEANUP_S3_BUCKET=simcore-volume-backups AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001 @@ -19,7 +19,7 @@ AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1 AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678 API_SERVER_DEV_FEATURES_ENABLED=0 -API_SERVER_LOGLEVEL=WARNING +API_SERVER_LOGLEVEL=INFO API_SERVER_PROFILING=1 TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT=25 @@ -29,7 +29,7 @@ AUTOSCALING_DOCKER_JOIN_DRAINED=True AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION=False AUTOSCALING_EC2_ACCESS=null AUTOSCALING_EC2_INSTANCES=null -AUTOSCALING_LOGLEVEL=WARNING +AUTOSCALING_LOGLEVEL=INFO AUTOSCALING_NODES_MONITORING=null AUTOSCALING_POLL_INTERVAL=10 AUTOSCALING_SSM_ACCESS=null @@ -39,7 +39,7 @@ AWS_S3_CLI_S3=null CATALOG_BACKGROUND_TASK_REST_TIME=60 CATALOG_DEV_FEATURES_ENABLED=0 CATALOG_HOST=catalog -CATALOG_LOGLEVEL=WARNING +CATALOG_LOGLEVEL=INFO CATALOG_PORT=8000 CATALOG_PROFILING=1 CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, "RAM": {"limit": 2147483648, "reservation": 2147483648}}' @@ -52,7 +52,7 @@ CLUSTERS_KEEPER_DASK_WORKER_SATURATION=inf CLUSTERS_KEEPER_EC2_ACCESS=null CLUSTERS_KEEPER_SSM_ACCESS=null CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX="" -CLUSTERS_KEEPER_LOGLEVEL=WARNING +CLUSTERS_KEEPER_LOGLEVEL=INFO CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION=5 CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null CLUSTERS_KEEPER_TASK_INTERVAL=30 @@ -60,7 +60,7 @@ CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null DASK_SCHEDULER_HOST=dask-scheduler DASK_SCHEDULER_PORT=8786 -DASK_SIDECAR_LOGLEVEL=WARNING +DASK_SIDECAR_LOGLEVEL=INFO DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem DASK_TLS_KEY=/home/scu/.dask/dask-key.pem @@ -91,7 +91,7 @@ DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REA DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL=0 DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' DIRECTOR_V2_HOST=director-v2 -DIRECTOR_V2_LOGLEVEL=WARNING +DIRECTOR_V2_LOGLEVEL=INFO DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH=null DIRECTOR_V2_PORT=8000 DIRECTOR_V2_PROFILING=1 @@ -115,7 +115,7 @@ FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.i # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password' INVITATIONS_DEFAULT_PRODUCT=osparc INVITATIONS_HOST=invitations -INVITATIONS_LOGLEVEL=WARNING +INVITATIONS_LOGLEVEL=INFO INVITATIONS_OSPARC_URL=http://127.0.0.1.nip.io:9081 INVITATIONS_PASSWORD=adminadmin INVITATIONS_PORT=8000 @@ -124,6 +124,7 @@ INVITATIONS_SWAGGER_API_DOC_ENABLED=1 INVITATIONS_USERNAME=admin LOG_FORMAT_LOCAL_DEV_ENABLED=1 +LOG_FILTER_MAPPING='{}' PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676 @@ -138,7 +139,7 @@ PAYMENTS_FAKE_COMPLETION=0 PAYMENTS_GATEWAY_API_SECRET=adminadmin PAYMENTS_GATEWAY_URL=http://127.0.0.1:32769 PAYMENTS_HOST=payments -PAYMENTS_LOGLEVEL=WARNING +PAYMENTS_LOGLEVEL=INFO PAYMENTS_PASSWORD=adminadmin PAYMENTS_PORT=8000 PAYMENTS_STRIPE_API_SECRET='REPLACE_ME_with_api_secret' @@ -179,7 +180,7 @@ RESOURCE_MANAGER_RESOURCE_TTL_S=900 RESOURCE_USAGE_TRACKER_HOST=resource-usage-tracker RESOURCE_USAGE_TRACKER_PORT=8000 RESOURCE_USAGE_TRACKER_EXTERNAL_PORT=8000 -RESOURCE_USAGE_TRACKER_LOGLEVEL=WARNING +RESOURCE_USAGE_TRACKER_LOGLEVEL=INFO RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED=1 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL=6 RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC=300 @@ -214,7 +215,7 @@ BF_API_KEY=none BF_API_SECRET=none STORAGE_ENDPOINT=storage:8080 STORAGE_HOST=storage -STORAGE_LOGLEVEL=WARNING +STORAGE_LOGLEVEL=INFO STORAGE_PORT=8080 STORAGE_PROFILING=1 # STORAGE ---- @@ -245,7 +246,7 @@ WB_GC_GARBAGE_COLLECTOR='{"GARBAGE_COLLECTOR_INTERVAL_S": 30}' WB_GC_GROUPS=0 WB_GC_INVITATIONS=null WB_GC_LOGIN=null -WB_GC_LOGLEVEL=WARNING +WB_GC_LOGLEVEL=INFO WB_GC_META_MODELING=0 WB_GC_NOTIFICATIONS=0 WB_GC_PAYMENTS=null @@ -278,7 +279,7 @@ WB_DB_EL_GARBAGE_COLLECTOR=null WB_DB_EL_GROUPS=0 WB_DB_EL_INVITATIONS=null WB_DB_EL_LOGIN=null -WB_DB_EL_LOGLEVEL=WARNING +WB_DB_EL_LOGLEVEL=INFO WB_DB_EL_META_MODELING=0 WB_DB_EL_NOTIFICATIONS=0 WB_DB_EL_PAYMENTS=null @@ -348,7 +349,7 @@ WEBSERVER_GROUPS=1 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180 WEBSERVER_HOST=webserver WEBSERVER_LOGIN={} -WEBSERVER_LOGLEVEL=WARNING +WEBSERVER_LOGLEVEL=INFO WEBSERVER_META_MODELING=1 WEBSERVER_NOTIFICATIONS=1 WEBSERVER_PAYMENTS={} diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 86bb38a08ca..9ffc6b32c5b 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Any, NotRequired, TypeAlias, TypedDict, TypeVar +from .logging_utils_filtering import GeneralLogFilter, LoggerName, MessageSubstring from .utils_secrets import mask_sensitive_data _logger = logging.getLogger(__name__) @@ -86,7 +87,11 @@ def format(self, record) -> str: # log_level=%{WORD:log_level} \| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \| log_source=%{DATA:log_source} \| log_msg=%{GREEDYDATA:log_msg} -def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None: +def config_all_loggers( + *, + log_format_local_dev_enabled: bool, + logger_filter_mapping: dict[LoggerName, list[MessageSubstring]], +) -> None: """ Applies common configuration to ALL registered loggers """ @@ -102,12 +107,25 @@ def config_all_loggers(*, log_format_local_dev_enabled: bool) -> None: fmt = LOCAL_FORMATTING for logger in loggers: - set_logging_handler( + _set_logging_handler( logger, fmt=fmt, log_format_local_dev_enabled=log_format_local_dev_enabled ) + for logger_name, filtered_routes in logger_filter_mapping.items(): + logger = logging.getLogger(logger_name) + # Check if the logger has any handlers or is in active use + if not logger.hasHandlers(): + _logger.warning( + "Logger %s does not have any handlers. Filter will not be added.", + logger_name, + ) + continue + + log_filter = GeneralLogFilter(filtered_routes) + logger.addFilter(log_filter) + -def set_logging_handler( +def _set_logging_handler( logger: logging.Logger, *, fmt: str, diff --git a/packages/service-library/src/servicelib/logging_utils_filtering.py b/packages/service-library/src/servicelib/logging_utils_filtering.py new file mode 100644 index 00000000000..8d40f501328 --- /dev/null +++ b/packages/service-library/src/servicelib/logging_utils_filtering.py @@ -0,0 +1,28 @@ +""" +This codes originates from this article + https://medium.com/swlh/add-log-decorators-to-your-python-project-84094f832181 + +SEE also https://github.com/Delgan/loguru for a future alternative +""" + +import logging +from typing import TypeAlias + +_logger = logging.getLogger(__name__) + +LoggerName: TypeAlias = str +MessageSubstring: TypeAlias = str + + +class GeneralLogFilter(logging.Filter): + def __init__(self, filtered_routes: list[str]) -> None: + super().__init__() + self.filtered_routes = filtered_routes + + def filter(self, record: logging.LogRecord) -> bool: + msg = record.getMessage() + + # Check if the filtered routes exists in the message + return not any( + filter_criteria in msg for filter_criteria in self.filtered_routes + ) diff --git a/packages/service-library/tests/test_logging_utils_filtering.py b/packages/service-library/tests/test_logging_utils_filtering.py new file mode 100644 index 00000000000..64084c3204a --- /dev/null +++ b/packages/service-library/tests/test_logging_utils_filtering.py @@ -0,0 +1,99 @@ +# pylint: disable=redefined-outer-name + +import logging +from typing import Generator + +import pytest +from servicelib.logging_utils_filtering import GeneralLogFilter + + +@pytest.fixture +def logger_with_filter() -> Generator[tuple[logging.Logger, list[str]], None, None]: + # Set up a logger for testing + logger = logging.getLogger("uvicorn.access") + logger.setLevel(logging.DEBUG) + + # Create a list to capture log outputs + log_capture = [] + + # Create a handler that appends log messages to the log_capture list + class ListHandler(logging.Handler): + def emit(self, record): + log_capture.append(self.format(record)) + + handler = ListHandler() + logger.addHandler(handler) + + # Set up the filter based on the new logic + filtered_routes = [ + '"GET / HTTP/1.1" 200', + '"GET /metrics HTTP/1.1" 200', + ] + + # Add the GeneralLogFilter to the logger + log_filter = GeneralLogFilter(filtered_routes) + logger.addFilter(log_filter) + + # Return logger and the log_capture for testing + yield logger, log_capture + + # Cleanup: remove handlers and filters after test + logger.handlers.clear() + logger.filters.clear() + + +def test_log_filtered_out(logger_with_filter: tuple[logging.Logger, list[str]]): + logger, log_capture = logger_with_filter + + # Create a log record that should be filtered out (matches the filter criteria) + record = logger.makeRecord( + name="uvicorn.access", + level=logging.INFO, + fn="testfile", + lno=10, + msg='"GET / HTTP/1.1" 200 OK', + args=(), + exc_info=None, + ) + logger.handle(record) + + # Assert no log messages were captured (filtered out) + assert len(log_capture) == 0 + + +def test_log_allowed(logger_with_filter): + logger, log_capture = logger_with_filter + + # Create a log record that should NOT be filtered out (doesn't match any filter criteria) + record = logger.makeRecord( + name="uvicorn.access", + level=logging.INFO, + fn="testfile", + lno=10, + msg='"GET /another HTTP/1.1" 200 OK', + args=(), + exc_info=None, + ) + logger.handle(record) + + # Assert the log message was captured (not filtered out) + assert len(log_capture) == 1 + + +def test_log_with_different_status(logger_with_filter): + logger, log_capture = logger_with_filter + + # Create a log record that has the same route but a different status code (should pass through) + record = logger.makeRecord( + name="uvicorn.access", + level=logging.INFO, + fn="testfile", + lno=10, + msg='"GET / HTTP/1.1" 500 Internal Server Error', + args=(), + exc_info=None, + ) + logger.handle(record) + + # Assert the log message was captured (not filtered out due to different status code) + assert len(log_capture) == 1 diff --git a/services/agent/src/simcore_service_agent/core/application.py b/services/agent/src/simcore_service_agent/core/application.py index 777c22a422c..41c80b07d61 100644 --- a/services/agent/src/simcore_service_agent/core/application.py +++ b/services/agent/src/simcore_service_agent/core/application.py @@ -30,7 +30,8 @@ def _setup_logger(settings: ApplicationSettings): logging.basicConfig(level=settings.LOGLEVEL.value) # NOSONAR logging.root.setLevel(settings.LOGLEVEL.value) config_all_loggers( - log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.AGENT_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.AGENT_VOLUMES_LOG_FILTER_MAPPING, ) diff --git a/services/agent/src/simcore_service_agent/core/settings.py b/services/agent/src/simcore_service_agent/core/settings.py index 96545d0355d..d901a34ca90 100644 --- a/services/agent/src/simcore_service_agent/core/settings.py +++ b/services/agent/src/simcore_service_agent/core/settings.py @@ -2,6 +2,7 @@ from models_library.basic_types import BootModeEnum, LogLevel from pydantic import AnyHttpUrl, Field, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.r_clone import S3Provider from settings_library.rabbit import RabbitSettings @@ -25,6 +26,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): "disabled if you want to have structured logs!" ), ) + AGENT_VOLUMES_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["AGENT_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) AGENT_VOLUMES_CLEANUP_TARGET_SWARM_STACK_NAME: str = Field( ..., description="Exactly the same as director-v2's `SWARM_STACK_NAME` env var" ) diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 39b05720a6d..04dcd397c28 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -52,7 +52,8 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.log_level) logging.root.setLevel(settings.log_level) config_all_loggers( - log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.API_SERVER_LOG_FILTER_MAPPING, ) _logger.debug("App settings:\n%s", settings.json(indent=2)) diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 3c00b3489b9..ab00d2d2ba8 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -3,6 +3,7 @@ from models_library.basic_types import BootModeEnum, LogLevel from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr from pydantic.class_validators import validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.director_v2 import DirectorV2Settings @@ -55,6 +56,11 @@ class BasicSettings(BaseCustomSettings, MixinLoggingSettings): env=["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + API_SERVER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["API_SERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) @validator("LOG_LEVEL", pre=True) @classmethod diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index 600a8cd507c..a9ba65945a9 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -22,6 +22,7 @@ root_validator, validator, ) +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings @@ -240,6 +241,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + AUTOSCALING_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) AUTOSCALING_EC2_ACCESS: AutoscalingEC2Settings | None = Field( auto_default_from_env=True diff --git a/services/autoscaling/src/simcore_service_autoscaling/main.py b/services/autoscaling/src/simcore_service_autoscaling/main.py index 90272bcb12f..c73f3bdf94c 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/main.py +++ b/services/autoscaling/src/simcore_service_autoscaling/main.py @@ -13,7 +13,8 @@ logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=the_settings.AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=the_settings.AUTOSCALING_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/catalog/src/simcore_service_catalog/core/settings.py b/services/catalog/src/simcore_service_catalog/core/settings.py index 6235dcfd37f..b07680b27cb 100644 --- a/services/catalog/src/simcore_service_catalog/core/settings.py +++ b/services/catalog/src/simcore_service_catalog/core/settings.py @@ -8,6 +8,7 @@ from models_library.basic_types import BootModeEnum, BuildTargetEnum, LogLevel from models_library.services_resources import ResourcesDict from pydantic import ByteSize, Field, PositiveInt, parse_obj_as +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.http_client_request import ClientRequestSettings from settings_library.postgres import PostgresSettings @@ -61,6 +62,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): env=["CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + CATALOG_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["CATALOG_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) CATALOG_DEV_FEATURES_ENABLED: bool = Field( default=False, description="Enables development features. WARNING: make sure it is disabled in production .env file!", diff --git a/services/catalog/src/simcore_service_catalog/main.py b/services/catalog/src/simcore_service_catalog/main.py index f5c9f4ee97c..ef94b84d5f1 100644 --- a/services/catalog/src/simcore_service_catalog/main.py +++ b/services/catalog/src/simcore_service_catalog/main.py @@ -14,7 +14,8 @@ logging.basicConfig(level=_the_settings.CATALOG_LOG_LEVEL.value) # NOSONAR logging.root.setLevel(_the_settings.CATALOG_LOG_LEVEL.value) config_all_loggers( - log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=_the_settings.CATALOG_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_the_settings.CATALOG_LOG_FILTER_MAPPING, ) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index ff2e74bbedc..98e8d5db004 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -20,6 +20,7 @@ parse_obj_as, validator, ) +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings @@ -267,6 +268,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + CLUSTERS_KEEPER_LOG_FILTER_MAPPING: dict[ + LoggerName, list[MessageSubstring] + ] = Field( + default_factory=dict, + env=["CLUSTERS_KEEPER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) CLUSTERS_KEEPER_EC2_ACCESS: ClustersKeeperEC2Settings | None = Field( auto_default_from_env=True diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py index 5e55931c36c..bcf872e807d 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/main.py @@ -13,7 +13,8 @@ logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=the_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=the_settings.CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=the_settings.CLUSTERS_KEEPER_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py index 80661c7ecb2..1ddc63de0b6 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py @@ -3,6 +3,7 @@ from models_library.basic_types import LogLevel from pydantic import Field, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.utils_logging import MixinLoggingSettings @@ -40,6 +41,11 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): env=["DASK_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + DASK_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["DASK_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) def as_scheduler(self) -> bool: return bool(self.DASK_START_AS_SCHEDULER) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py index 79dfd08cbdb..5a4496aecdd 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/tasks.py @@ -63,7 +63,8 @@ async def dask_setup(worker: distributed.Worker) -> None: for handler in logging.getLogger("distributed").handlers: logging.getLogger("distributed").removeHandler(handler) config_all_loggers( - log_format_local_dev_enabled=settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.DASK_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.DASK_LOG_FILTER_MAPPING, ) logger.info("Setting up worker...") diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py index 8c4fb44e8e9..8ee59f8e24e 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/application.py @@ -40,7 +40,8 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.LOG_LEVEL.value) logging.root.setLevel(settings.LOG_LEVEL.value) config_all_loggers( - log_format_local_dev_enabled=settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.DATCORE_ADAPTER_LOG_FILTER_MAPPING, ) # keep mostly quiet noisy loggers diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py index 68e879807ab..6c6d06b3043 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py @@ -3,6 +3,7 @@ from models_library.basic_types import BootModeEnum, LogLevel from pydantic import Field, parse_obj_as, validator from pydantic.networks import AnyUrl +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.tracing import TracingSettings from settings_library.utils_logging import MixinLoggingSettings @@ -40,6 +41,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + DATCORE_ADAPTER_LOG_FILTER_MAPPING: dict[ + LoggerName, list[MessageSubstring] + ] = Field( + default_factory=dict, + env=["DATCORE_ADAPTER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) DATCORE_ADAPTER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DATCORE_ADAPTER_TRACING: TracingSettings | None = Field( auto_default_from_env=True, description="settings for opentelemetry tracing" diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 330717e6062..9972a42cce5 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -113,7 +113,8 @@ def create_base_app(settings: AppSettings | None = None) -> FastAPI: logging.basicConfig(level=settings.LOG_LEVEL.value) logging.root.setLevel(settings.LOG_LEVEL.value) config_all_loggers( - log_format_local_dev_enabled=settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.DIRECTOR_V2_LOG_FILTER_MAPPING, ) _logger.debug(settings.json(indent=2)) diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index d495dd4aeef..7cb52904799 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -20,6 +20,7 @@ NoAuthentication, ) from pydantic import AnyHttpUrl, AnyUrl, Field, NonNegativeInt, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.docker_registry import RegistrySettings @@ -131,6 +132,11 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + DIRECTOR_V2_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["DIRECTOR_V2_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: bool = Field( diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 0b1b7f460cd..45e843ad712 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -14,6 +14,7 @@ services: environment: API_SERVER_DEV_FEATURES_ENABLED: ${API_SERVER_DEV_FEATURES_ENABLED} API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + API_SERVER_LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} API_SERVER_LOGLEVEL: ${API_SERVER_LOGLEVEL} API_SERVER_PROFILING: ${API_SERVER_PROFILING} @@ -110,6 +111,7 @@ services: SSM_REGION_NAME: ${SSM_REGION_NAME} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -155,6 +157,7 @@ services: DIRECTOR_HOST: ${DIRECTOR_HOST:-director} DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_HOST: ${POSTGRES_HOST} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} @@ -196,6 +199,7 @@ services: CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY} CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: ${CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: ${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} PRIMARY_EC2_INSTANCES_ALLOWED_TYPES: ${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES} PRIMARY_EC2_INSTANCES_KEY_NAME: ${PRIMARY_EC2_INSTANCES_KEY_NAME} @@ -323,6 +327,7 @@ services: DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} MONITORING_ENABLED: ${MONITORING_ENABLED} @@ -403,6 +408,7 @@ services: - default environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -445,6 +451,7 @@ services: INVITATIONS_SWAGGER_API_DOC_ENABLED: ${INVITATIONS_SWAGGER_API_DOC_ENABLED} INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} payments: @@ -455,6 +462,7 @@ services: - default environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES: ${PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES} PAYMENTS_ACCESS_TOKEN_SECRET_KEY: ${PAYMENTS_ACCESS_TOKEN_SECRET_KEY} PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} @@ -499,6 +507,7 @@ services: - default environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} POSTGRES_HOST: ${POSTGRES_HOST} @@ -535,6 +544,7 @@ services: - default environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -619,6 +629,7 @@ services: WEBSERVER_PROFILING: ${WEBSERVER_PROFILING} WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + WEBSERVER_LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} # WEBSERVER_SERVER_HOST @@ -855,6 +866,7 @@ services: GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} SWARM_STACK_NAME: ${SWARM_STACK_NAME} SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} WEBSERVER_ACTIVITY: ${WB_DB_EL_ACTIVITY} @@ -942,6 +954,7 @@ services: GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} STORAGE_HOST: ${STORAGE_HOST} STORAGE_PORT: ${STORAGE_PORT} @@ -1015,6 +1028,7 @@ services: environment: AGENT_LOGLEVEL: ${AGENT_LOGLEVEL} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: ${AGENT_VOLUMES_CLEANUP_S3_ENDPOINT} AGENT_VOLUMES_CLEANUP_S3_REGION: ${AGENT_VOLUMES_CLEANUP_S3_REGION} AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY: ${AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY} @@ -1048,6 +1062,7 @@ services: DASK_TLS_CERT: ${DASK_TLS_CERT} DASK_SCHEDULER_HOST: ${DASK_SCHEDULER_HOST:-dask-scheduler} DASK_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + DASK_LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} DASK_SIDECAR_LOGLEVEL: ${DASK_SIDECAR_LOGLEVEL} SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: ${SWARM_STACK_NAME}_computational_shared_data SIDECAR_COMP_SERVICES_SHARED_FOLDER: ${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data} @@ -1075,6 +1090,7 @@ services: - storage_subnet environment: DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + DATCORE_ADAPTER_LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} @@ -1087,6 +1103,7 @@ services: BF_API_SECRET: ${BF_API_SECRET} DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} POSTGRES_HOST: ${POSTGRES_HOST} diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index a60ccc504cd..3bf448d0892 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,6 +1,7 @@ import datetime from pydantic import Field, parse_obj_as, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.director_v2 import DirectorV2Settings @@ -34,6 +35,13 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: dict[ + LoggerName, list[MessageSubstring] + ] = Field( + default_factory=dict, + env=["DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: datetime.timedelta = Field( default=datetime.timedelta(minutes=60), diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index a2de28aa723..55b8513d7e9 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -14,7 +14,8 @@ logging.basicConfig(level=_the_settings.DYNAMIC_SCHEDULER_LOGLEVEL.value) logging.root.setLevel(_the_settings.DYNAMIC_SCHEDULER_LOGLEVEL.value) config_all_loggers( - log_format_local_dev_enabled=_the_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=_the_settings.DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_the_settings.DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 8592afd2440..ce5f48a8b21 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -116,7 +116,8 @@ def setup_logger(settings: ApplicationSettings): logging.basicConfig(level=settings.log_level) logging.root.setLevel(settings.log_level) config_all_loggers( - log_format_local_dev_enabled=settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings.DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.DY_SIDECAR_LOG_FILTER_MAPPING, ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index 4e151e29c00..cf6f636622a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -12,6 +12,7 @@ from models_library.services import DynamicServiceKey, RunID, ServiceVersion from models_library.users import UserID from pydantic import ByteSize, Field, PositiveInt, parse_obj_as, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.aws_s3_cli import AwsS3CliSettings from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings @@ -133,6 +134,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): env=["DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + DY_SIDECAR_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default={}, + env=["DY_SIDECAR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) DY_SIDECAR_USER_ID: UserID DY_SIDECAR_PROJECT_ID: ProjectID DY_SIDECAR_NODE_ID: NodeID diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py index 7856d991a43..b672dfa7ce2 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/settings.py @@ -10,6 +10,7 @@ VersionTag, ) from pydantic import ByteSize, Field, PositiveInt, parse_obj_as, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.efs import AwsEfsSettings from settings_library.postgres import PostgresSettings @@ -83,6 +84,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + EFS_GUARDIAN_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["EFS_GUARDIAN_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) EFS_GUARDIAN_AWS_EFS_SETTINGS: AwsEfsSettings = Field(auto_default_from_env=True) EFS_GUARDIAN_POSTGRES: PostgresSettings = Field(auto_default_from_env=True) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/main.py b/services/efs-guardian/src/simcore_service_efs_guardian/main.py index dd107472181..f155d24f0f8 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/main.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/main.py @@ -13,7 +13,8 @@ logging.basicConfig(level=the_settings.log_level) logging.root.setLevel(the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=the_settings.EFS_GUARDIAN_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=the_settings.EFS_GUARDIAN_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/invitations/src/simcore_service_invitations/core/settings.py b/services/invitations/src/simcore_service_invitations/core/settings.py index 35bdd32ab31..43ff3130562 100644 --- a/services/invitations/src/simcore_service_invitations/core/settings.py +++ b/services/invitations/src/simcore_service_invitations/core/settings.py @@ -2,6 +2,7 @@ from models_library.products import ProductName from pydantic import Field, HttpUrl, PositiveInt, SecretStr, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.tracing import TracingSettings @@ -49,6 +50,11 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + INVITATIONS_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["INVITATIONS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) @cached_property def LOG_LEVEL(self): diff --git a/services/invitations/src/simcore_service_invitations/main.py b/services/invitations/src/simcore_service_invitations/main.py index bddf5d2d3eb..407e239c1bd 100644 --- a/services/invitations/src/simcore_service_invitations/main.py +++ b/services/invitations/src/simcore_service_invitations/main.py @@ -15,7 +15,8 @@ logging.basicConfig(level=the_settings.log_level) # NOSONAR logging.root.setLevel(the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=the_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=the_settings.INVITATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=the_settings.INVITATIONS_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index b56c79322cb..7efa14c1aaa 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -10,6 +10,7 @@ parse_obj_as, validator, ) +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag from settings_library.email import SMTPSettings @@ -43,6 +44,11 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + PAYMENTS_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["PAYMENTS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) @cached_property def LOG_LEVEL(self): # noqa: N802 diff --git a/services/payments/src/simcore_service_payments/main.py b/services/payments/src/simcore_service_payments/main.py index a39832f283d..6ff09676f1f 100644 --- a/services/payments/src/simcore_service_payments/main.py +++ b/services/payments/src/simcore_service_payments/main.py @@ -15,7 +15,8 @@ logging.basicConfig(level=_the_settings.log_level) # NOSONAR logging.root.setLevel(_the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=_the_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=_the_settings.PAYMENTS_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_the_settings.PAYMENTS_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 7658d1da0cd..bca6e4dac55 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -3,6 +3,7 @@ from models_library.basic_types import BootModeEnum from pydantic import Field, PositiveInt, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.postgres import PostgresSettings @@ -59,6 +60,13 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING: dict[ + LoggerName, list[MessageSubstring] + ] = Field( + default_factory=dict, + env=["RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) @cached_property def LOG_LEVEL(self) -> LogLevel: # noqa: N802 diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py index b0f4e3a7fe2..05890bd7a51 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/web_main.py @@ -15,7 +15,8 @@ logging.basicConfig(level=the_settings.log_level) # NOSONAR logging.root.setLevel(the_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=the_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=the_settings.RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=the_settings.RESOURCE_USAGE_TRACKER_LOG_FILTER_MAPPING, ) # SINGLETON FastAPI app diff --git a/services/storage/src/simcore_service_storage/cli.py b/services/storage/src/simcore_service_storage/cli.py index 6a9e209b0da..d50d540d5b4 100644 --- a/services/storage/src/simcore_service_storage/cli.py +++ b/services/storage/src/simcore_service_storage/cli.py @@ -26,7 +26,8 @@ def run(): logging.basicConfig(level=settings_obj.log_level) logging.root.setLevel(settings_obj.log_level) config_all_loggers( - log_format_local_dev_enabled=settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED + log_format_local_dev_enabled=settings_obj.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings_obj.STORAGE_LOG_FILTER_MAPPING, ) # keep mostly quiet noisy loggers diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index dc01d96f99e..5a847da1556 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -1,6 +1,7 @@ from typing import Any from pydantic import Field, PositiveInt, root_validator, validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.basic_types import LogLevel, PortInt from settings_library.postgres import PostgresSettings @@ -68,6 +69,11 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): env=["STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + STORAGE_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) @validator("LOG_LEVEL") @classmethod diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 617447c134a..9fa0ac5de7c 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -14,6 +14,7 @@ from pydantic import AnyHttpUrl, parse_obj_as, root_validator, validator from pydantic.fields import Field, ModelField from pydantic.types import PositiveInt +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings from settings_library.email import SMTPSettings from settings_library.postgres import PostgresSettings @@ -114,6 +115,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): env=["WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) + WEBSERVER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( + default_factory=dict, + env=["WEBSERVER_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"], + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ) # TODO: find a better name!? WEBSERVER_SERVER_HOST: str = Field( default="0.0.0.0", # nosec diff --git a/services/web/server/src/simcore_service_webserver/cli.py b/services/web/server/src/simcore_service_webserver/cli.py index bbd9b327a9b..ca85670ad74 100644 --- a/services/web/server/src/simcore_service_webserver/cli.py +++ b/services/web/server/src/simcore_service_webserver/cli.py @@ -55,6 +55,7 @@ def _setup_app_from_settings( level=settings.log_level, slow_duration=settings.AIODEBUG_SLOW_DURATION_SECS, log_format_local_dev_enabled=settings.WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.WEBSERVER_LOG_FILTER_MAPPING, ) app = create_application() diff --git a/services/web/server/src/simcore_service_webserver/log.py b/services/web/server/src/simcore_service_webserver/log.py index 6635518d091..ee6957732f4 100644 --- a/services/web/server/src/simcore_service_webserver/log.py +++ b/services/web/server/src/simcore_service_webserver/log.py @@ -27,14 +27,18 @@ def setup_logging( *, level: str | int, slow_duration: float | None = None, - log_format_local_dev_enabled: bool + log_format_local_dev_enabled: bool, + logger_filter_mapping: dict, ): # service log level logging.basicConfig(level=level) # root logging.root.setLevel(level) - config_all_loggers(log_format_local_dev_enabled=log_format_local_dev_enabled) + config_all_loggers( + log_format_local_dev_enabled=log_format_local_dev_enabled, + logger_filter_mapping=logger_filter_mapping, + ) # Enforces same log-level to aiohttp & gunicorn access loggers # diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py index d80957cca89..a074c4d77e1 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/conftest.py @@ -53,7 +53,9 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc ) # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging(level=logging.DEBUG, log_format_local_dev_enabled=True) + setup_logging( + level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + ) plugin_settings = StudiesDispatcherSettings.create_from_envs() print(plugin_settings.json(indent=1)) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py index 3a0e2513c7f..2a26d924622 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py @@ -67,7 +67,9 @@ def app_cfg(default_app_cfg, unused_tcp_port_factory, monkeypatch) -> dict[str, cfg[section]["enabled"] = False # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging(level=logging.DEBUG, log_format_local_dev_enabled=True) + setup_logging( + level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + ) # Enforces smallest GC in the background task cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index f412a99d56c..ed8a2c2979f 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -121,7 +121,9 @@ def app_cfg( cfg[section]["enabled"] = False # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging(level=logging.DEBUG, log_format_local_dev_enabled=True) + setup_logging( + level=logging.DEBUG, log_format_local_dev_enabled=True, logger_filter_mapping={} + ) # Enforces smallest GC in the background task cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 From cdbaf0cd7aa3adca0e8ab9a44bacfffe8b8ae644 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:47:02 +0100 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8E=A8=F0=9F=90=9B=20[Frontend]=20Red?= =?UTF-8?q?uce=20the=20initial=20number=20of=20``patch``=20calls=20(#6641)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../class/osparc/dashboard/GridButtonBase.js | 16 +++-------- .../class/osparc/dashboard/StudyBrowser.js | 5 ++-- .../osparc/dashboard/StudyBrowserHeader.js | 27 ++++++++++--------- .../source/class/osparc/data/model/Study.js | 14 ++++------ .../source/class/osparc/widget/NodeOutputs.js | 25 +++++++++++++++++ .../osparc/workbench/DiskUsageController.js | 6 ++--- .../osparc/workbench/DiskUsageIndicator.js | 6 ++--- 7 files changed, 57 insertions(+), 42 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js index 04be2898461..ad0a78c20c1 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/GridButtonBase.js @@ -77,33 +77,26 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { MODIFIED: { row: 0, column: 0, - rowSpan: 1, - colSpan: 2 }, UI_MODE: { row: 0, - column: 3, - colSpan: 1 + column: 1, }, UPDATES: { row: 0, - column: 4, - colSpan: 1 + column: 2, }, TSR: { row: 1, column: 0, - colSpan: 2 }, HITS: { row: 1, - column: 2, - colSpan: 1 + column: 1, }, PERMISSION: { row: 1, - column: 4, - colSpan: 1 + column: 2, } } }, @@ -173,7 +166,6 @@ qx.Class.define("osparc.dashboard.GridButtonBase", { case "footer": { const fgrid = new qx.ui.layout.Grid(); fgrid.setSpacing(2); - fgrid.setRowFlex(2, 1); fgrid.setColumnFlex(0, 1); control = new qx.ui.container.Composite().set({ backgroundColor: "background-card-overlay", diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js index 70a3554ae41..804986735a7 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowser.js @@ -924,8 +924,9 @@ qx.Class.define("osparc.dashboard.StudyBrowser", { if (filterData.text) { this.__changeContext("search"); } else { - // Back to My Workspace - this.__changeContext("studiesAndFolders", null, null); + const workspaceId = this.getCurrentWorkspaceId(); + const folderId = this.getCurrentFolderId(); + this.__changeContext("studiesAndFolders", workspaceId, folderId); } }); } diff --git a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js index 54235a08c45..36cb6375e26 100644 --- a/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js +++ b/services/static-webserver/client/source/class/osparc/dashboard/StudyBrowserHeader.js @@ -185,9 +185,21 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { return control || this.base(arguments, id); }, + __titleTapped: function() { + const workspaceId = this.getCurrentWorkspaceId(); + const folderId = null; + this.setCurrentFolderId(folderId); + this.fireDataEvent("locationChanged", { + workspaceId, + folderId, + }); + }, + __buildLayout: function() { this.getChildControl("icon"); const title = this.getChildControl("workspace-title"); + title.resetCursor(); + title.removeListener("tap", this.__titleTapped, this); this.getChildControl("breadcrumbs"); this.getChildControl("edit-button").exclude(); this.resetAccessRights(); @@ -198,27 +210,16 @@ qx.Class.define("osparc.dashboard.StudyBrowserHeader", { this.__setIcon("@FontAwesome5Solid/search/24"); title.set({ value: this.tr("Search results"), - cursor: "auto", }); } else if (currentContext === "workspaces") { this.__setIcon(osparc.store.Workspaces.iconPath(32)); title.set({ value: this.tr("Shared Workspaces"), - cursor: "auto", }) } else if (currentContext === "studiesAndFolders") { const workspaceId = this.getCurrentWorkspaceId(); - title.set({ - cursor: "pointer" - }); - title.addListener("tap", () => { - const folderId = null; - this.setCurrentFolderId(folderId); - this.fireDataEvent("locationChanged", { - workspaceId, - folderId, - }); - }); + title.setCursor("pointer"); + title.addListener("tap", this.__titleTapped, this); const workspace = osparc.store.Workspaces.getInstance().getWorkspace(workspaceId); if (workspace) { const thumbnail = workspace.getThumbnail(); diff --git a/services/static-webserver/client/source/class/osparc/data/model/Study.js b/services/static-webserver/client/source/class/osparc/data/model/Study.js index 912f838486f..598e0575d22 100644 --- a/services/static-webserver/client/source/class/osparc/data/model/Study.js +++ b/services/static-webserver/client/source/class/osparc/data/model/Study.js @@ -82,14 +82,14 @@ qx.Class.define("osparc.data.model.Study", { workspaceId: { check: "Number", - init: true, + init: null, nullable: true, event: "changeWorkspaceId" }, folderId: { check: "Number", - init: true, + init: null, nullable: true, event: "changeFolderId" }, @@ -103,9 +103,9 @@ qx.Class.define("osparc.data.model.Study", { description: { check: "String", - nullable: false, + nullable: true, event: "changeDescription", - init: "" + init: null }, prjOwner: { @@ -356,11 +356,7 @@ qx.Class.define("osparc.data.model.Study", { jsonObject[key] = this.getUi().serialize(); return; } - const value = this.get(key); - if (value !== null) { - // only put the value in the payload if there is a value - jsonObject[key] = value; - } + jsonObject[key] = this.get(key); }); return jsonObject; }, diff --git a/services/static-webserver/client/source/class/osparc/widget/NodeOutputs.js b/services/static-webserver/client/source/class/osparc/widget/NodeOutputs.js index 071db993e30..c8da28a0c48 100644 --- a/services/static-webserver/client/source/class/osparc/widget/NodeOutputs.js +++ b/services/static-webserver/client/source/class/osparc/widget/NodeOutputs.js @@ -164,10 +164,34 @@ qx.Class.define("osparc.widget.NodeOutputs", { for (let i=0; i { if ("resp" in presignedLinkData && presignedLinkData.resp) { diff --git a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js index 5510906f2bb..c0a3a8e835d 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js +++ b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageController.js @@ -119,10 +119,10 @@ qx.Class.define("osparc.workbench.DiskUsageController", { let freeSpace = osparc.utils.Utils.bytesToSize(diskHostUsage.free); let warningLevel = this.__getWarningLevel(diskHostUsage.free); - if ("STATE_VOLUMES" in data.usage) { - const diskVolsUsage = data.usage["STATE_VOLUMES"]; + if ("STATES_VOLUMES" in data.usage) { + const diskVolsUsage = data.usage["STATES_VOLUMES"]; if (diskVolsUsage["used_percent"] > diskHostUsage["used_percent"]) { - // "STATE_VOLUMES" is more critical so it takes over + // "STATES_VOLUMES" is more critical so it takes over freeSpace = osparc.utils.Utils.bytesToSize(diskVolsUsage.free); warningLevel = this.__getWarningLevel(diskVolsUsage.free); } diff --git a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageIndicator.js b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageIndicator.js index 496b94079cb..173b3689524 100644 --- a/services/static-webserver/client/source/class/osparc/workbench/DiskUsageIndicator.js +++ b/services/static-webserver/client/source/class/osparc/workbench/DiskUsageIndicator.js @@ -160,10 +160,10 @@ qx.Class.define("osparc.workbench.DiskUsageIndicator", { let progress = `${diskHostUsage["used_percent"]}%`; let labelDiskSize = osparc.utils.Utils.bytesToSize(diskHostUsage.free); let toolTipText = this.tr("Disk usage"); - if ("STATE_VOLUMES" in diskUsage["usage"]) { - const diskVolsUsage = diskUsage["usage"]["STATE_VOLUMES"]; + if ("STATES_VOLUMES" in diskUsage["usage"]) { + const diskVolsUsage = diskUsage["usage"]["STATES_VOLUMES"]; if (diskVolsUsage["used_percent"] > diskHostUsage["used_percent"]) { - // "STATE_VOLUMES" is more critical so it takes over + // "STATES_VOLUMES" is more critical so it takes over color1 = this.__getIndicatorColor(diskVolsUsage.free); progress = `${diskVolsUsage["used_percent"]}%`; labelDiskSize = osparc.utils.Utils.bytesToSize(diskVolsUsage.free); From 0bc2185ff3dc419695cfb2ee7115ae4ede2b5c20 Mon Sep 17 00:00:00 2001 From: Odei Maiz <33152403+odeimaiz@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:47:25 +0100 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20[Frontend]=20Fix:=20Selected?= =?UTF-8?q?=20Pricing=20Unit=20bgColor=20(#6646)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dustin Kaiser <8209087+mrnicegyu11@users.noreply.github.com> --- .../class/osparc/service/PricingUnitsList.js | 22 +++---------- .../source/class/osparc/study/PricingUnits.js | 33 +++++++++++++------ 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js b/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js index fac14e0449d..f7dcc85a457 100644 --- a/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js +++ b/services/static-webserver/client/source/class/osparc/service/PricingUnitsList.js @@ -64,30 +64,16 @@ qx.Class.define("osparc.service.PricingUnitsList", { __populateList: function(pricingUnits) { this.getChildControl("pricing-units-container").removeAll(); - if (pricingUnits.length === 0) { + if (pricingUnits.length) { + const pUnits = new osparc.study.PricingUnits(pricingUnits, null, false); + this.getChildControl("pricing-units-container").add(pUnits); + } else { const notFound = new qx.ui.basic.Label().set({ value: this.tr("No Tiers found"), font: "text-14" }); this.getChildControl("pricing-units-container").add(notFound); - return; } - - pricingUnits.forEach(pricingUnit => { - const pUnit = new osparc.study.PricingUnit(pricingUnit).set({ - allowGrowY: false - }); - this.getChildControl("pricing-units-container").add(pUnit); - }); - - const buttons = this.getChildControl("pricing-units-container").getChildren(); - const keepDefaultSelected = () => { - buttons.forEach(btn => { - btn.setValue(btn.getUnitData().isDefault()); - }); - }; - keepDefaultSelected(); - buttons.forEach(btn => btn.addListener("execute", () => keepDefaultSelected())); } } }); diff --git a/services/static-webserver/client/source/class/osparc/study/PricingUnits.js b/services/static-webserver/client/source/class/osparc/study/PricingUnits.js index 2f3fe92d2ac..5238825a838 100644 --- a/services/static-webserver/client/source/class/osparc/study/PricingUnits.js +++ b/services/static-webserver/client/source/class/osparc/study/PricingUnits.js @@ -18,14 +18,15 @@ qx.Class.define("osparc.study.PricingUnits", { extend: qx.ui.container.Composite, - construct: function(pricingUnits, preselectedPricingUnit) { + construct: function(pricingUnits, preselectedPricingUnit, changeSelectionAllowed = true) { this.base(arguments); this.set({ - layout: new qx.ui.layout.HBox(5) + layout: new qx.ui.layout.HBox(5), + allowGrowY: false, }); - this.__buildLayout(pricingUnits, preselectedPricingUnit); + this.__buildLayout(pricingUnits, preselectedPricingUnit, changeSelectionAllowed); }, properties: { @@ -38,7 +39,7 @@ qx.Class.define("osparc.study.PricingUnits", { }, members: { - __buildLayout: function(pricingUnits, preselectedPricingUnit) { + __buildLayout: function(pricingUnits, preselectedPricingUnit, changeSelectionAllowed) { const buttons = []; pricingUnits.forEach(pricingUnit => { const button = new osparc.study.PricingUnit(pricingUnit); @@ -47,7 +48,12 @@ qx.Class.define("osparc.study.PricingUnits", { }); const groupOptions = new qx.ui.form.RadioGroup(); - buttons.forEach(btn => groupOptions.add(btn)); + buttons.forEach(btn => { + groupOptions.add(btn); + btn.bind("value", btn, "backgroundColor", { + converter: selected => selected ? "background-main-1" : "transparent" + }); + }); if (preselectedPricingUnit) { const buttonFound = buttons.find(button => button.getUnitData().getPricingUnitId() === preselectedPricingUnit["pricingUnitId"]); @@ -63,12 +69,19 @@ qx.Class.define("osparc.study.PricingUnits", { }); } - buttons.forEach(button => button.addListener("changeValue", e => { - if (e.getData()) { - const selectedUnitId = button.getUnitData().getPricingUnitId(); - this.setSelectedUnitId(selectedUnitId); + buttons.forEach(button => { + if (!changeSelectionAllowed) { + button.setCursor("default"); } - })); + button.addListener("execute", () => { + if (changeSelectionAllowed) { + const selectedUnitId = button.getUnitData().getPricingUnitId(); + this.setSelectedUnitId(selectedUnitId); + } else { + buttons.forEach(btn => btn.setValue(btn.getUnitData().isDefault())); + } + }); + }); } } });