From 44090213c83e95348b0d44a34e34424d6bd12c74 Mon Sep 17 00:00:00 2001 From: Steve Bunting Date: Fri, 12 Jan 2024 11:35:29 -0800 Subject: [PATCH 1/5] make all the things strings --- src/supergood/client.py | 11 ++++++----- src/supergood/helpers.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/supergood/client.py b/src/supergood/client.py index b5dc83e..694e4f4 100644 --- a/src/supergood/client.py +++ b/src/supergood/client.py @@ -13,7 +13,7 @@ from .api import Api from .constants import * -from .helpers import redact_values, safe_decode, safe_parse_json +from .helpers import decode_headers, redact_values, safe_decode, safe_parse_json from .logger import Logger from .remote_config import get_vendor_endpoint_from_config, parse_remote_config_json from .repeating_thread import RepeatingThread @@ -158,6 +158,7 @@ def _should_ignore( def _cache_request(self, request_id, url, method, body, headers): request = {} try: + url = safe_decode(url) # we do this first so the urlparse isn't also bytes host_domain = urlparse(url).hostname request["metadata"] = {} # Check that we should cache the request @@ -178,11 +179,11 @@ def _cache_request(self, request_id, url, method, body, headers): filtered_headers = ( {} if (not self.base_config["logRequestHeaders"] or headers is None) - else dict(headers) + else decode_headers(headers) ) request["request"] = { "id": request_id, - "method": method, + "method": safe_decode(method), "url": url, "body": filtered_body, "headers": filtered_headers, @@ -216,13 +217,13 @@ def _cache_response( filtered_headers = ( {} if not self.base_config["logResponseHeaders"] - else dict(response_headers) + else decode_headers(dict(response_headers)) ) response = { "body": filtered_body, "headers": filtered_headers, "status": response_status, - "statusText": response_status_text, + "statusText": safe_decode(response_status_text), "respondedAt": datetime.now().isoformat(), } self._response_cache[request_id] = { diff --git a/src/supergood/helpers.py b/src/supergood/helpers.py index 0945271..3306ae9 100644 --- a/src/supergood/helpers.py +++ b/src/supergood/helpers.py @@ -264,10 +264,19 @@ def safe_decode(input, encoding="utf-8"): try: if isinstance(input, bytes) and input[:2] == GZIP_START_BYTES: return gzip.decompress(input) - if isinstance(input, str): return input - return input.decode(encoding) except Exception: return str(input) + + +def decode_headers(headers, encoding="utf-8"): + for key, value in headers.items(): + decoded_key = key + if isinstance(key, bytes): + decoded_key = safe_decode(key, encoding) + decoded_value = value + if isinstance(value, bytes): + decoded_value = safe_decode(value, encoding) + headers[decoded_key] = decoded_value From 80dc26b734bf86c3eb9b9fd89f4bca0dbd0a2f72 Mon Sep 17 00:00:00 2001 From: Steve Bunting Date: Fri, 12 Jan 2024 11:45:54 -0800 Subject: [PATCH 2/5] add more reporting on error --- src/supergood/client.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/supergood/client.py b/src/supergood/client.py index 694e4f4..51123b0 100644 --- a/src/supergood/client.py +++ b/src/supergood/client.py @@ -325,7 +325,21 @@ def flush_cache(self, force=False) -> None: self.log.debug(f"Flushing {len(data)} items") self.api.post_events(data) except Exception: - payload = self._build_log_payload() + was_set = False + try: + urls = [] + for entry in data: + if entry.get("request", None): + urls.append(entry.get("request").get("url")) + num_events = len(data) + was_set = True + except Exception: + # something really messed up, just report out + pass + if was_set: + payload = self._build_log_payload(num_events=num_events, urls=urls) + else: + payload = self._build_log_payload() trace = "".join(traceback.format_exc()) self.log.error(ERRORS["POSTING_EVENTS"], trace, payload) finally: # always occurs, even from internal returns From 6ed392be96928d9b748a9f7236046794419ea90a Mon Sep 17 00:00:00 2001 From: Steve Bunting Date: Fri, 12 Jan 2024 13:54:16 -0800 Subject: [PATCH 3/5] fix: forgot to dict-ify request headers --- src/supergood/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/supergood/client.py b/src/supergood/client.py index 51123b0..db49378 100644 --- a/src/supergood/client.py +++ b/src/supergood/client.py @@ -179,7 +179,7 @@ def _cache_request(self, request_id, url, method, body, headers): filtered_headers = ( {} if (not self.base_config["logRequestHeaders"] or headers is None) - else decode_headers(headers) + else decode_headers(dict(headers)) ) request["request"] = { "id": request_id, From b1d59040ef6009981fdb04623027cf9ac30647c6 Mon Sep 17 00:00:00 2001 From: Steve Bunting Date: Fri, 12 Jan 2024 13:58:34 -0800 Subject: [PATCH 4/5] fix dict update --- src/supergood/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/supergood/helpers.py b/src/supergood/helpers.py index 3306ae9..0b74ab9 100644 --- a/src/supergood/helpers.py +++ b/src/supergood/helpers.py @@ -272,6 +272,7 @@ def safe_decode(input, encoding="utf-8"): def decode_headers(headers, encoding="utf-8"): + new_headers = {} for key, value in headers.items(): decoded_key = key if isinstance(key, bytes): @@ -279,4 +280,5 @@ def decode_headers(headers, encoding="utf-8"): decoded_value = value if isinstance(value, bytes): decoded_value = safe_decode(value, encoding) - headers[decoded_key] = decoded_value + new_headers[decoded_key] = decoded_value + return new_headers From b1bd8ecb69dd446ad25fef1497e6495c7631f900 Mon Sep 17 00:00:00 2001 From: Steve Bunting Date: Fri, 12 Jan 2024 14:04:25 -0800 Subject: [PATCH 5/5] better error logging --- src/supergood/client.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/supergood/client.py b/src/supergood/client.py index db49378..01083e7 100644 --- a/src/supergood/client.py +++ b/src/supergood/client.py @@ -325,22 +325,17 @@ def flush_cache(self, force=False) -> None: self.log.debug(f"Flushing {len(data)} items") self.api.post_events(data) except Exception: - was_set = False + trace = "".join(traceback.format_exc()) try: urls = [] for entry in data: if entry.get("request", None): urls.append(entry.get("request").get("url")) num_events = len(data) - was_set = True - except Exception: - # something really messed up, just report out - pass - if was_set: payload = self._build_log_payload(num_events=num_events, urls=urls) - else: + except Exception: + # something is really messed up, just report out payload = self._build_log_payload() - trace = "".join(traceback.format_exc()) self.log.error(ERRORS["POSTING_EVENTS"], trace, payload) finally: # always occurs, even from internal returns for response_key in response_keys: