From ee0cdc548519554bfd65e314cf229df5a5f491d7 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 11 Oct 2024 08:52:35 -0500 Subject: [PATCH 1/4] fix: do not send duplicated events in batch --- event_routing_backends/backends/events_router.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/event_routing_backends/backends/events_router.py b/event_routing_backends/backends/events_router.py index 93511adf..42c3dd7e 100644 --- a/event_routing_backends/backends/events_router.py +++ b/event_routing_backends/backends/events_router.py @@ -155,8 +155,13 @@ def bulk_send(self, events): for events_for_route in event_routes.values(): prepared_events = [] host = None + ids = set() for _, updated_event, host, _ in events_for_route: + if updated_event["id"] in set(): + logger.info(f"Found duplicated event {updated_event['id']}") + continue prepared_events.append(updated_event) + ids.add(updated_event["id"]) if prepared_events: # pragma: no cover self.dispatch_bulk_events( From 7be30341bb4b5ab4ee3ad5ae4d3080e2eb741855 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 15 Oct 2024 13:31:20 -0500 Subject: [PATCH 2/4] test: add id to fixtures --- event_routing_backends/backends/tests/test_events_router.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/event_routing_backends/backends/tests/test_events_router.py b/event_routing_backends/backends/tests/test_events_router.py index fec2bac0..6cd3a584 100644 --- a/event_routing_backends/backends/tests/test_events_router.py +++ b/event_routing_backends/backends/tests/test_events_router.py @@ -77,6 +77,7 @@ def setUp(self): } self.transformed_event = { + 'id': 'some-random-uuid', 'name': str(sentinel.name), 'transformed': True, 'event_time': '2020-01-01T12:12:12.000000+00:00', @@ -126,6 +127,7 @@ def setUp(self): self.bulk_transformed_events = [ { + 'id': 'some-uuid-1', 'name': str(sentinel.name), 'transformed': True, 'event_time': '2020-01-01T12:12:12.000000+00:00', @@ -134,6 +136,7 @@ def setUp(self): }, }, { + 'id': 'some-uuid-2', 'name': str(sentinel.name), 'transformed': True, 'event_time': '2020-01-01T12:12:12.000000+00:01', @@ -142,6 +145,7 @@ def setUp(self): }, }, { + 'id': 'some-uuid-3', 'name': str(sentinel.name), 'transformed': True, 'event_time': '2020-01-01T12:12:12.000000+00:02', From 4036988a72bf7e7ec460a8d90dd5371af0359ae2 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 15 Oct 2024 14:14:22 -0500 Subject: [PATCH 3/4] test: add tests for duplicated statements --- event_routing_backends/backends/events_router.py | 2 +- .../backends/tests/test_events_router.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/event_routing_backends/backends/events_router.py b/event_routing_backends/backends/events_router.py index 42c3dd7e..6d6965b9 100644 --- a/event_routing_backends/backends/events_router.py +++ b/event_routing_backends/backends/events_router.py @@ -157,7 +157,7 @@ def bulk_send(self, events): host = None ids = set() for _, updated_event, host, _ in events_for_route: - if updated_event["id"] in set(): + if updated_event["id"] in ids: logger.info(f"Found duplicated event {updated_event['id']}") continue prepared_events.append(updated_event) diff --git a/event_routing_backends/backends/tests/test_events_router.py b/event_routing_backends/backends/tests/test_events_router.py index 6cd3a584..e8cacbed 100644 --- a/event_routing_backends/backends/tests/test_events_router.py +++ b/event_routing_backends/backends/tests/test_events_router.py @@ -144,6 +144,15 @@ def setUp(self): 'key': 'value 1' }, }, + { + 'id': 'some-uuid-3', + 'name': str(sentinel.name), + 'transformed': True, + 'event_time': '2020-01-01T12:12:12.000000+00:02', + 'data': { + 'key': 'value 2' + }, + }, { 'id': 'some-uuid-3', 'name': str(sentinel.name), @@ -845,6 +854,7 @@ def test_successful_routing_of_bulk_events( router.bulk_send(self.bulk_transformed_events) overridden_events = self.bulk_transformed_events.copy() + overridden_events.pop() for event in overridden_events: event['new_key'] = 'new_value' @@ -1149,6 +1159,7 @@ def test_successful_routing_of_bulk_events( router.bulk_send(self.bulk_transformed_events) overridden_events = self.bulk_transformed_events.copy() + overridden_events.pop() for event in overridden_events: event['new_key'] = 'new_value' From 821a54171885e43a3594887733907916fde86966 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 15 Oct 2024 14:23:38 -0500 Subject: [PATCH 4/4] chore: add comments for bulk duplicated test --- event_routing_backends/backends/tests/test_events_router.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/event_routing_backends/backends/tests/test_events_router.py b/event_routing_backends/backends/tests/test_events_router.py index e8cacbed..25a016b2 100644 --- a/event_routing_backends/backends/tests/test_events_router.py +++ b/event_routing_backends/backends/tests/test_events_router.py @@ -854,6 +854,7 @@ def test_successful_routing_of_bulk_events( router.bulk_send(self.bulk_transformed_events) overridden_events = self.bulk_transformed_events.copy() + # Remove duplicated events from expected call overridden_events.pop() for event in overridden_events: @@ -1159,6 +1160,7 @@ def test_successful_routing_of_bulk_events( router.bulk_send(self.bulk_transformed_events) overridden_events = self.bulk_transformed_events.copy() + # Remove duplicated events from expected call overridden_events.pop() for event in overridden_events: