From 98891ff67052a49b080665c777bf431939a792f3 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Tue, 26 Mar 2024 16:00:12 +0000 Subject: [PATCH 1/8] fix: Added environment specific labels to client library when running in Cloud Run Jobs --- .../handlers/_monitored_resources.py | 46 +++++++++++++++- google/cloud/logging_v2/handlers/handlers.py | 15 ++---- .../handlers/test__monitored_resources.py | 52 ++++++++++++++----- 3 files changed, 87 insertions(+), 26 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index f93d54988..d607796f7 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os from google.cloud.logging_v2.resource import Resource @@ -67,6 +68,20 @@ _PROJECT_NAME = "project/project-id" """Attribute in metadata server when in GKE environment.""" +_GAE_RESOURCE_TYPE = "gae_app" +"""Resource type for App Engine environment.""" + +_CLOUD_RUN_JOB_RESOURCE_TYPE = "cloud_run_job" +"""Resource type for Cloud Run Jobs.""" + +_GAE_TRACE_ID_LABEL = "appengine.googleapis.com/trace_id" +"""Extra trace label to be added on App Engine environments""" + +_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL = "run.googleapis.com/execution_name" +_CLOUD_RUN_JOBS_TASK_INDEX_LABEL = "run.googleapis.com/task_index" +_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL = "run.googleapis.com/task_attempt" +"""Extra labels for Cloud Run environments to be recognized by Cloud Run Jobs web UI.""" + def _create_functions_resource(): """Create a standardized Cloud Functions resource. @@ -159,7 +174,7 @@ def _create_cloud_run_job_resource(): region = retrieve_metadata_server(_REGION_ID) project = retrieve_metadata_server(_PROJECT_NAME) resource = Resource( - type="cloud_run_job", + type=_CLOUD_RUN_JOB_RESOURCE_TYPE, labels={ "project_id": project if project else "", "job_name": os.environ.get(_CLOUD_RUN_JOB_ID, ""), @@ -177,7 +192,7 @@ def _create_app_engine_resource(): zone = retrieve_metadata_server(_ZONE_ID) project = retrieve_metadata_server(_PROJECT_NAME) resource = Resource( - type="gae_app", + type=_GAE_RESOURCE_TYPE, labels={ "project_id": project if project else "", "module_id": os.environ.get(_GAE_SERVICE_ENV, ""), @@ -233,3 +248,30 @@ def detect_resource(project=""): else: # use generic global resource return _create_global_resource(project) + + +def add_environment_labels(resource: Resource, record: logging.LogRecord): + """Returns additional labels to be appended on to a LogRecord object based on the + local environment. Defaults to an empty dictionary if none apply. This is only to be + used for CloudLoggingHandler, as the structured logging daemon already does this. + + Args: + resource (google.cloud.logging.Resource): Resource based on the environment + record (logging.LogRecord): A LogRecord object representing a log record + Returns: + Dict[str, str]: New labels to append to the labels of the LogRecord + """ + additional_labels = {} + def set_item(key, val): + if val: + additional_labels[key] = val + + if resource: + if resource.type == _GAE_RESOURCE_TYPE: + set_item(_GAE_TRACE_ID_LABEL, record._trace) + elif resource.type == _CLOUD_RUN_JOB_RESOURCE_TYPE: + set_item(_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, os.environ.get(_CLOUD_RUN_EXECUTION_ID, "")) + set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, os.environ.get(_CLOUD_RUN_TASK_INDEX, "")) + set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, "")) + + return additional_labels diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 3d6ab9d1e..b834c5b66 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -19,7 +19,9 @@ import logging from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport -from google.cloud.logging_v2.handlers._monitored_resources import detect_resource +from google.cloud.logging_v2.handlers._monitored_resources import ( + detect_resource, add_environment_labels +) from google.cloud.logging_v2.handlers._helpers import get_request_data DEFAULT_LOGGER_NAME = "python" @@ -40,12 +42,6 @@ """These environments require us to remove extra handlers on setup""" _CLEAR_HANDLER_RESOURCE_TYPES = ("gae_app", "cloud_function") -"""Extra trace label to be added on App Engine environments""" -_GAE_TRACE_ID_LABEL = "appengine.googleapis.com/trace_id" - -"""Resource name for App Engine environments""" -_GAE_RESOURCE_TYPE = "gae_app" - class CloudLoggingFilter(logging.Filter): """Python standard ``logging`` Filter class to add Cloud Logging @@ -206,9 +202,8 @@ def emit(self, record): labels = record._labels message = _format_and_parse_message(record, self) - if resource.type == _GAE_RESOURCE_TYPE and record._trace is not None: - # add GAE-specific label - labels = {_GAE_TRACE_ID_LABEL: record._trace, **(labels or {})} + labels = {**add_environment_labels(resource, record), **(labels or {})} or None + # send off request self.transport.send( record, diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index e788f8e34..cf3a78adb 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -14,32 +14,22 @@ import unittest +import logging import mock import os import functools -from google.cloud.logging_v2.handlers._monitored_resources import ( - _create_functions_resource, -) from google.cloud.logging_v2.handlers._monitored_resources import ( _create_app_engine_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_functions_resource, _create_kubernetes_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_cloud_run_service_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_cloud_run_job_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_compute_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_global_resource, + detect_resource, + add_environment_labels ) -from google.cloud.logging_v2.handlers._monitored_resources import detect_resource from google.cloud.logging_v2.handlers import _monitored_resources from google.cloud.logging_v2.resource import Resource @@ -353,3 +343,37 @@ def test_detect_partial_data(self): # project id not returned from metadata serve # should be empty string self.assertEqual(resource.labels["project_id"], "") + + +class Test_Add_Environmental_Labels(Test_Resource_Detection): + + def setUp(self): + super().setUp() + self.record = logging.LogRecord("logname", None, None, None, "test", None, None) + + def add_environment_labels_test_flow(self, resource_type, expected_labels): + resource = Resource(type=resource_type, labels={}) + actual_labels = add_environment_labels(resource, self.record) + self.assertDictEqual(actual_labels, expected_labels) + + def test_gae_label(self): + trace_id = "trace_id" + setattr(self.record, "_trace", trace_id) + self.add_environment_labels_test_flow(_monitored_resources._GAE_RESOURCE_TYPE, { + _monitored_resources._GAE_TRACE_ID_LABEL: trace_id + }) + + def test_cloud_run_job_label(self): + test_execution_id = "test_job_12345" + test_task_index = "1" + test_task_attempt = "12" + + os.environ[_monitored_resources._CLOUD_RUN_EXECUTION_ID] = test_execution_id + os.environ[_monitored_resources._CLOUD_RUN_TASK_INDEX] = test_task_index + os.environ[_monitored_resources._CLOUD_RUN_TASK_ATTEMPT] = test_task_attempt + + self.add_environment_labels_test_flow(_monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, { + _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: test_execution_id, + _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: test_task_index, + _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: test_task_attempt + }) From 1400010ba06ce93ccc50f52c5354a6b730cd62aa Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 3 Apr 2024 20:24:31 +0000 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../handlers/_monitored_resources.py | 16 ++++++++++--- google/cloud/logging_v2/handlers/handlers.py | 3 ++- .../handlers/test__monitored_resources.py | 23 +++++++++++-------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index d607796f7..93959cf21 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -262,6 +262,7 @@ def add_environment_labels(resource: Resource, record: logging.LogRecord): Dict[str, str]: New labels to append to the labels of the LogRecord """ additional_labels = {} + def set_item(key, val): if val: additional_labels[key] = val @@ -270,8 +271,17 @@ def set_item(key, val): if resource.type == _GAE_RESOURCE_TYPE: set_item(_GAE_TRACE_ID_LABEL, record._trace) elif resource.type == _CLOUD_RUN_JOB_RESOURCE_TYPE: - set_item(_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, os.environ.get(_CLOUD_RUN_EXECUTION_ID, "")) - set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, os.environ.get(_CLOUD_RUN_TASK_INDEX, "")) - set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, "")) + set_item( + _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, + os.environ.get(_CLOUD_RUN_EXECUTION_ID, ""), + ) + set_item( + _CLOUD_RUN_JOBS_TASK_INDEX_LABEL, + os.environ.get(_CLOUD_RUN_TASK_INDEX, ""), + ) + set_item( + _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, + os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, ""), + ) return additional_labels diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index b834c5b66..750b8b13a 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -20,7 +20,8 @@ from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.handlers._monitored_resources import ( - detect_resource, add_environment_labels + detect_resource, + add_environment_labels, ) from google.cloud.logging_v2.handlers._helpers import get_request_data diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index cf3a78adb..5654a4a76 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -28,7 +28,7 @@ _create_compute_resource, _create_global_resource, detect_resource, - add_environment_labels + add_environment_labels, ) from google.cloud.logging_v2.handlers import _monitored_resources from google.cloud.logging_v2.resource import Resource @@ -346,7 +346,6 @@ def test_detect_partial_data(self): class Test_Add_Environmental_Labels(Test_Resource_Detection): - def setUp(self): super().setUp() self.record = logging.LogRecord("logname", None, None, None, "test", None, None) @@ -359,9 +358,10 @@ def add_environment_labels_test_flow(self, resource_type, expected_labels): def test_gae_label(self): trace_id = "trace_id" setattr(self.record, "_trace", trace_id) - self.add_environment_labels_test_flow(_monitored_resources._GAE_RESOURCE_TYPE, { - _monitored_resources._GAE_TRACE_ID_LABEL: trace_id - }) + self.add_environment_labels_test_flow( + _monitored_resources._GAE_RESOURCE_TYPE, + {_monitored_resources._GAE_TRACE_ID_LABEL: trace_id}, + ) def test_cloud_run_job_label(self): test_execution_id = "test_job_12345" @@ -372,8 +372,11 @@ def test_cloud_run_job_label(self): os.environ[_monitored_resources._CLOUD_RUN_TASK_INDEX] = test_task_index os.environ[_monitored_resources._CLOUD_RUN_TASK_ATTEMPT] = test_task_attempt - self.add_environment_labels_test_flow(_monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, { - _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: test_execution_id, - _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: test_task_index, - _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: test_task_attempt - }) + self.add_environment_labels_test_flow( + _monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, + { + _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: test_execution_id, + _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: test_task_index, + _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: test_task_attempt, + }, + ) From d0a49a14f55be9320c885edd12b0eafafbc1fccd Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Wed, 3 Apr 2024 20:20:02 +0000 Subject: [PATCH 3/8] Removed unnecessary import --- google/cloud/logging_v2/handlers/_monitored_resources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 93959cf21..e235b610b 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os from google.cloud.logging_v2.resource import Resource From 85ca7f23bae1c99ee72a76274d5cfd56921fa230 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Tue, 9 Apr 2024 20:14:51 +0000 Subject: [PATCH 4/8] Changed unit tests to pytest --- .../handlers/_monitored_resources.py | 43 +++++------ .../handlers/test__monitored_resources.py | 74 +++++++++++-------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index e235b610b..b08fae15a 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os from google.cloud.logging_v2.resource import Resource @@ -81,6 +82,8 @@ _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL = "run.googleapis.com/task_attempt" """Extra labels for Cloud Run environments to be recognized by Cloud Run Jobs web UI.""" +_ADDITIONAL_ENV_LABELS = None +"""Additional environmental labels placeholder, to be only evaluated once""" def _create_functions_resource(): """Create a standardized Cloud Functions resource. @@ -260,27 +263,19 @@ def add_environment_labels(resource: Resource, record: logging.LogRecord): Returns: Dict[str, str]: New labels to append to the labels of the LogRecord """ - additional_labels = {} - - def set_item(key, val): - if val: - additional_labels[key] = val - - if resource: - if resource.type == _GAE_RESOURCE_TYPE: - set_item(_GAE_TRACE_ID_LABEL, record._trace) - elif resource.type == _CLOUD_RUN_JOB_RESOURCE_TYPE: - set_item( - _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, - os.environ.get(_CLOUD_RUN_EXECUTION_ID, ""), - ) - set_item( - _CLOUD_RUN_JOBS_TASK_INDEX_LABEL, - os.environ.get(_CLOUD_RUN_TASK_INDEX, ""), - ) - set_item( - _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, - os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, ""), - ) - - return additional_labels + global _ADDITIONAL_ENV_LABELS + if _ADDITIONAL_ENV_LABELS is None: + _ADDITIONAL_ENV_LABELS = {} + def set_item(key, val): + if val: + _ADDITIONAL_ENV_LABELS[key] = val + + if resource: + if resource.type == _GAE_RESOURCE_TYPE: + set_item(_GAE_TRACE_ID_LABEL, record._trace) + elif resource.type == _CLOUD_RUN_JOB_RESOURCE_TYPE: + set_item(_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, os.environ.get(_CLOUD_RUN_EXECUTION_ID, "")) + set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, os.environ.get(_CLOUD_RUN_TASK_INDEX, "")) + set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, "")) + + return _ADDITIONAL_ENV_LABELS diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index 5654a4a76..babef612e 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest import unittest import logging @@ -345,38 +346,51 @@ def test_detect_partial_data(self): self.assertEqual(resource.labels["project_id"], "") -class Test_Add_Environmental_Labels(Test_Resource_Detection): - def setUp(self): - super().setUp() - self.record = logging.LogRecord("logname", None, None, None, "test", None, None) - - def add_environment_labels_test_flow(self, resource_type, expected_labels): - resource = Resource(type=resource_type, labels={}) - actual_labels = add_environment_labels(resource, self.record) - self.assertDictEqual(actual_labels, expected_labels) - - def test_gae_label(self): - trace_id = "trace_id" - setattr(self.record, "_trace", trace_id) - self.add_environment_labels_test_flow( +@pytest.mark.parametrize( + "resource_type,os_environ,record_attrs,expected_labels", + [ + ( _monitored_resources._GAE_RESOURCE_TYPE, - {_monitored_resources._GAE_TRACE_ID_LABEL: trace_id}, - ) - - def test_cloud_run_job_label(self): - test_execution_id = "test_job_12345" - test_task_index = "1" - test_task_attempt = "12" - - os.environ[_monitored_resources._CLOUD_RUN_EXECUTION_ID] = test_execution_id - os.environ[_monitored_resources._CLOUD_RUN_TASK_INDEX] = test_task_index - os.environ[_monitored_resources._CLOUD_RUN_TASK_ATTEMPT] = test_task_attempt - - self.add_environment_labels_test_flow( + {}, + {"_trace": "trace_id"}, + {_monitored_resources._GAE_TRACE_ID_LABEL: "trace_id"} + ), + ( _monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, { - _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: test_execution_id, - _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: test_task_index, - _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: test_task_attempt, + _monitored_resources._CLOUD_RUN_EXECUTION_ID: "test_job_12345", + _monitored_resources._CLOUD_RUN_TASK_INDEX: "1", + _monitored_resources._CLOUD_RUN_TASK_ATTEMPT: "12", }, + {}, + { + _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: "test_job_12345", + _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: "1", + _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: "12", + } + ), + ( + "global", + {}, + {}, + {} ) + ] +) +def test_add_environment_labels(resource_type, os_environ, record_attrs, expected_labels): + os.environ.clear() + record = logging.LogRecord("logname", None, None, None, "test", None, None) + + resource = Resource(type=resource_type, labels={}) + + for attr, val in record_attrs.items(): + setattr(record, attr, val) + + os.environ.update(os_environ) + + labels = add_environment_labels(resource, record) + + try: + assert expected_labels == labels + finally: + _monitored_resources._ADDITIONAL_ENV_LABELS = None From 55e92682f777167686bfbf592c7f68dd94ece945 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Thu, 11 Apr 2024 14:29:46 +0000 Subject: [PATCH 5/8] Renamed add_environmental_labels to add_resource_labels; cached portions of add_resource_labels --- .../handlers/_monitored_resources.py | 60 +++++++++++++------ google/cloud/logging_v2/handlers/handlers.py | 4 +- .../handlers/test__monitored_resources.py | 11 ++-- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index b08fae15a..1f06ca308 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import logging import os @@ -82,8 +83,6 @@ _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL = "run.googleapis.com/task_attempt" """Extra labels for Cloud Run environments to be recognized by Cloud Run Jobs web UI.""" -_ADDITIONAL_ENV_LABELS = None -"""Additional environmental labels placeholder, to be only evaluated once""" def _create_functions_resource(): """Create a standardized Cloud Functions resource. @@ -252,7 +251,35 @@ def detect_resource(project=""): return _create_global_resource(project) -def add_environment_labels(resource: Resource, record: logging.LogRecord): +@functools.lru_cache(maxsize=None) +def _get_environmental_labels(resource_type): + """Builds a dictionary of labels to be inserted into a LogRecord of the given resource type. + This function should only build a dict of items that are consistent across multiple logging statements + of the same resource type, such as environment variables. + + Returns: + dict: + A dict representation of labels and the values of those labels + """ + labels = {} + + def set_item(key, environ_var): + val = os.environ.get(environ_var, "") + if val: + labels[key] = val + + if resource_type == _CLOUD_RUN_JOB_RESOURCE_TYPE: + set_item( + _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, + _CLOUD_RUN_EXECUTION_ID, + ) + set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, _CLOUD_RUN_TASK_INDEX) + set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, _CLOUD_RUN_TASK_ATTEMPT) + + return labels + + +def add_resource_labels(resource: Resource, record: logging.LogRecord): """Returns additional labels to be appended on to a LogRecord object based on the local environment. Defaults to an empty dictionary if none apply. This is only to be used for CloudLoggingHandler, as the structured logging daemon already does this. @@ -263,19 +290,14 @@ def add_environment_labels(resource: Resource, record: logging.LogRecord): Returns: Dict[str, str]: New labels to append to the labels of the LogRecord """ - global _ADDITIONAL_ENV_LABELS - if _ADDITIONAL_ENV_LABELS is None: - _ADDITIONAL_ENV_LABELS = {} - def set_item(key, val): - if val: - _ADDITIONAL_ENV_LABELS[key] = val - - if resource: - if resource.type == _GAE_RESOURCE_TYPE: - set_item(_GAE_TRACE_ID_LABEL, record._trace) - elif resource.type == _CLOUD_RUN_JOB_RESOURCE_TYPE: - set_item(_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, os.environ.get(_CLOUD_RUN_EXECUTION_ID, "")) - set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, os.environ.get(_CLOUD_RUN_TASK_INDEX, "")) - set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, os.environ.get(_CLOUD_RUN_TASK_ATTEMPT, "")) - - return _ADDITIONAL_ENV_LABELS + if not resource: + return None + + # Get environmental labels from the resource type + labels = _get_environmental_labels(resource.type) + + # Add labels from log record + if resource.type == _GAE_RESOURCE_TYPE and record._trace is not None: + labels[_GAE_TRACE_ID_LABEL] = record._trace + + return labels diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 750b8b13a..06e131442 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -21,7 +21,7 @@ from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.handlers._monitored_resources import ( detect_resource, - add_environment_labels, + add_resource_labels, ) from google.cloud.logging_v2.handlers._helpers import get_request_data @@ -203,7 +203,7 @@ def emit(self, record): labels = record._labels message = _format_and_parse_message(record, self) - labels = {**add_environment_labels(resource, record), **(labels or {})} or None + labels = {**add_resource_labels(resource, record), **(labels or {})} or None # send off request self.transport.send( diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index babef612e..42fd83b71 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -29,7 +29,7 @@ _create_compute_resource, _create_global_resource, detect_resource, - add_environment_labels, + add_resource_labels, ) from google.cloud.logging_v2.handlers import _monitored_resources from google.cloud.logging_v2.resource import Resource @@ -377,7 +377,7 @@ def test_detect_partial_data(self): ) ] ) -def test_add_environment_labels(resource_type, os_environ, record_attrs, expected_labels): +def test_add_resource_labels(resource_type, os_environ, record_attrs, expected_labels): os.environ.clear() record = logging.LogRecord("logname", None, None, None, "test", None, None) @@ -388,9 +388,6 @@ def test_add_environment_labels(resource_type, os_environ, record_attrs, expecte os.environ.update(os_environ) - labels = add_environment_labels(resource, record) + labels = add_resource_labels(resource, record) - try: - assert expected_labels == labels - finally: - _monitored_resources._ADDITIONAL_ENV_LABELS = None + assert expected_labels == labels From b9daf751de6e5409dec88fffb68f76b82ae0d5d5 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 11 Apr 2024 14:33:01 +0000 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/handlers/test__monitored_resources.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index 42fd83b71..28f064b7b 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -353,7 +353,7 @@ def test_detect_partial_data(self): _monitored_resources._GAE_RESOURCE_TYPE, {}, {"_trace": "trace_id"}, - {_monitored_resources._GAE_TRACE_ID_LABEL: "trace_id"} + {_monitored_resources._GAE_TRACE_ID_LABEL: "trace_id"}, ), ( _monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, @@ -367,15 +367,10 @@ def test_detect_partial_data(self): _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: "test_job_12345", _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: "1", _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: "12", - } + }, ), - ( - "global", - {}, - {}, - {} - ) - ] + ("global", {}, {}, {}), + ], ) def test_add_resource_labels(resource_type, os_environ, record_attrs, expected_labels): os.environ.clear() @@ -385,7 +380,7 @@ def test_add_resource_labels(resource_type, os_environ, record_attrs, expected_l for attr, val in record_attrs.items(): setattr(record, attr, val) - + os.environ.update(os_environ) labels = add_resource_labels(resource, record) From 7053648c68cea0ca91d02a5207a958d34fb5fad7 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Tue, 16 Apr 2024 20:06:31 +0000 Subject: [PATCH 7/8] Updated comments and _get_environmental_labels --- .../handlers/_monitored_resources.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 1f06ca308..dcc0aaedf 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -255,26 +255,26 @@ def detect_resource(project=""): def _get_environmental_labels(resource_type): """Builds a dictionary of labels to be inserted into a LogRecord of the given resource type. This function should only build a dict of items that are consistent across multiple logging statements - of the same resource type, such as environment variables. + of the same resource type, such as environment variables. Th Returns: dict: A dict representation of labels and the values of those labels """ labels = {} - - def set_item(key, environ_var): - val = os.environ.get(environ_var, "") - if val: - labels[key] = val - - if resource_type == _CLOUD_RUN_JOB_RESOURCE_TYPE: - set_item( - _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL, - _CLOUD_RUN_EXECUTION_ID, - ) - set_item(_CLOUD_RUN_JOBS_TASK_INDEX_LABEL, _CLOUD_RUN_TASK_INDEX) - set_item(_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL, _CLOUD_RUN_TASK_ATTEMPT) + environ_vars = { + _CLOUD_RUN_JOB_RESOURCE_TYPE: { + _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: _CLOUD_RUN_EXECUTION_ID, + _CLOUD_RUN_JOBS_TASK_INDEX_LABEL: _CLOUD_RUN_TASK_INDEX, + _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: _CLOUD_RUN_TASK_ATTEMPT + } + } + + if resource_type in environ_vars: + for key, env_var in environ_vars[resource_type].items(): + val = os.environ.get(env_var, "") + if val: + labels[key] = val return labels From e0c9c9a9994d7c05c3fc183d5c41768d24e79f3b Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 16 Apr 2024 20:09:37 +0000 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/logging_v2/handlers/_monitored_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index dcc0aaedf..5240fe746 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -266,7 +266,7 @@ def _get_environmental_labels(resource_type): _CLOUD_RUN_JOB_RESOURCE_TYPE: { _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: _CLOUD_RUN_EXECUTION_ID, _CLOUD_RUN_JOBS_TASK_INDEX_LABEL: _CLOUD_RUN_TASK_INDEX, - _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: _CLOUD_RUN_TASK_ATTEMPT + _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: _CLOUD_RUN_TASK_ATTEMPT, } }