diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index f93d54988..5240fe746 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools +import logging import os from google.cloud.logging_v2.resource import Resource @@ -67,6 +69,20 @@ _PROJECT_NAME = "project/project-id" """Attribute in metadata server when in GKE environment.""" +_GAE_RESOURCE_TYPE = "gae_app" +"""Resource type for App Engine environment.""" + +_CLOUD_RUN_JOB_RESOURCE_TYPE = "cloud_run_job" +"""Resource type for Cloud Run Jobs.""" + +_GAE_TRACE_ID_LABEL = "appengine.googleapis.com/trace_id" +"""Extra trace label to be added on App Engine environments""" + +_CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL = "run.googleapis.com/execution_name" +_CLOUD_RUN_JOBS_TASK_INDEX_LABEL = "run.googleapis.com/task_index" +_CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL = "run.googleapis.com/task_attempt" +"""Extra labels for Cloud Run environments to be recognized by Cloud Run Jobs web UI.""" + def _create_functions_resource(): """Create a standardized Cloud Functions resource. @@ -159,7 +175,7 @@ def _create_cloud_run_job_resource(): region = retrieve_metadata_server(_REGION_ID) project = retrieve_metadata_server(_PROJECT_NAME) resource = Resource( - type="cloud_run_job", + type=_CLOUD_RUN_JOB_RESOURCE_TYPE, labels={ "project_id": project if project else "", "job_name": os.environ.get(_CLOUD_RUN_JOB_ID, ""), @@ -177,7 +193,7 @@ def _create_app_engine_resource(): zone = retrieve_metadata_server(_ZONE_ID) project = retrieve_metadata_server(_PROJECT_NAME) resource = Resource( - type="gae_app", + type=_GAE_RESOURCE_TYPE, labels={ "project_id": project if project else "", "module_id": os.environ.get(_GAE_SERVICE_ENV, ""), @@ -233,3 +249,55 @@ def detect_resource(project=""): else: # use generic global resource return _create_global_resource(project) + + +@functools.lru_cache(maxsize=None) +def _get_environmental_labels(resource_type): + """Builds a dictionary of labels to be inserted into a LogRecord of the given resource type. + This function should only build a dict of items that are consistent across multiple logging statements + of the same resource type, such as environment variables. Th + + Returns: + dict: + A dict representation of labels and the values of those labels + """ + labels = {} + environ_vars = { + _CLOUD_RUN_JOB_RESOURCE_TYPE: { + _CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: _CLOUD_RUN_EXECUTION_ID, + _CLOUD_RUN_JOBS_TASK_INDEX_LABEL: _CLOUD_RUN_TASK_INDEX, + _CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: _CLOUD_RUN_TASK_ATTEMPT, + } + } + + if resource_type in environ_vars: + for key, env_var in environ_vars[resource_type].items(): + val = os.environ.get(env_var, "") + if val: + labels[key] = val + + return labels + + +def add_resource_labels(resource: Resource, record: logging.LogRecord): + """Returns additional labels to be appended on to a LogRecord object based on the + local environment. Defaults to an empty dictionary if none apply. This is only to be + used for CloudLoggingHandler, as the structured logging daemon already does this. + + Args: + resource (google.cloud.logging.Resource): Resource based on the environment + record (logging.LogRecord): A LogRecord object representing a log record + Returns: + Dict[str, str]: New labels to append to the labels of the LogRecord + """ + if not resource: + return None + + # Get environmental labels from the resource type + labels = _get_environmental_labels(resource.type) + + # Add labels from log record + if resource.type == _GAE_RESOURCE_TYPE and record._trace is not None: + labels[_GAE_TRACE_ID_LABEL] = record._trace + + return labels diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 3d6ab9d1e..06e131442 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -19,7 +19,10 @@ import logging from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport -from google.cloud.logging_v2.handlers._monitored_resources import detect_resource +from google.cloud.logging_v2.handlers._monitored_resources import ( + detect_resource, + add_resource_labels, +) from google.cloud.logging_v2.handlers._helpers import get_request_data DEFAULT_LOGGER_NAME = "python" @@ -40,12 +43,6 @@ """These environments require us to remove extra handlers on setup""" _CLEAR_HANDLER_RESOURCE_TYPES = ("gae_app", "cloud_function") -"""Extra trace label to be added on App Engine environments""" -_GAE_TRACE_ID_LABEL = "appengine.googleapis.com/trace_id" - -"""Resource name for App Engine environments""" -_GAE_RESOURCE_TYPE = "gae_app" - class CloudLoggingFilter(logging.Filter): """Python standard ``logging`` Filter class to add Cloud Logging @@ -206,9 +203,8 @@ def emit(self, record): labels = record._labels message = _format_and_parse_message(record, self) - if resource.type == _GAE_RESOURCE_TYPE and record._trace is not None: - # add GAE-specific label - labels = {_GAE_TRACE_ID_LABEL: record._trace, **(labels or {})} + labels = {**add_resource_labels(resource, record), **(labels or {})} or None + # send off request self.transport.send( record, diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index e788f8e34..28f064b7b 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -12,34 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest import unittest +import logging import mock import os import functools -from google.cloud.logging_v2.handlers._monitored_resources import ( - _create_functions_resource, -) from google.cloud.logging_v2.handlers._monitored_resources import ( _create_app_engine_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_functions_resource, _create_kubernetes_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_cloud_run_service_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_cloud_run_job_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_compute_resource, -) -from google.cloud.logging_v2.handlers._monitored_resources import ( _create_global_resource, + detect_resource, + add_resource_labels, ) -from google.cloud.logging_v2.handlers._monitored_resources import detect_resource from google.cloud.logging_v2.handlers import _monitored_resources from google.cloud.logging_v2.resource import Resource @@ -353,3 +344,45 @@ def test_detect_partial_data(self): # project id not returned from metadata serve # should be empty string self.assertEqual(resource.labels["project_id"], "") + + +@pytest.mark.parametrize( + "resource_type,os_environ,record_attrs,expected_labels", + [ + ( + _monitored_resources._GAE_RESOURCE_TYPE, + {}, + {"_trace": "trace_id"}, + {_monitored_resources._GAE_TRACE_ID_LABEL: "trace_id"}, + ), + ( + _monitored_resources._CLOUD_RUN_JOB_RESOURCE_TYPE, + { + _monitored_resources._CLOUD_RUN_EXECUTION_ID: "test_job_12345", + _monitored_resources._CLOUD_RUN_TASK_INDEX: "1", + _monitored_resources._CLOUD_RUN_TASK_ATTEMPT: "12", + }, + {}, + { + _monitored_resources._CLOUD_RUN_JOBS_EXECUTION_NAME_LABEL: "test_job_12345", + _monitored_resources._CLOUD_RUN_JOBS_TASK_INDEX_LABEL: "1", + _monitored_resources._CLOUD_RUN_JOBS_TASK_ATTEMPT_LABEL: "12", + }, + ), + ("global", {}, {}, {}), + ], +) +def test_add_resource_labels(resource_type, os_environ, record_attrs, expected_labels): + os.environ.clear() + record = logging.LogRecord("logname", None, None, None, "test", None, None) + + resource = Resource(type=resource_type, labels={}) + + for attr, val in record_attrs.items(): + setattr(record, attr, val) + + os.environ.update(os_environ) + + labels = add_resource_labels(resource, record) + + assert expected_labels == labels