Merge pull request #726 from rohanpm/compressed-config

Write compressed configuration [RHELDST-25461]

exodus_gw/aws/dynamodb.py

@@ -1,5 +1,7 @@
+import gzip
 import json
 import logging
+from base64 import b64decode, b64encode
 from datetime import datetime
 from itertools import islice
 from threading import Lock
@@ -94,7 +96,15 @@ def query_definitions(self) -> dict[str, Any]:
         )
         if query_result.get("Items"):
             item = query_result["Items"][0]
-            out = json.loads(item["config"]["S"])
+            if item_encoded := item["config"].get("B"):
+                # new-style: config is compressed and stored as bytes
+                item_bytes = b64decode(item_encoded)
+                item_json = gzip.decompress(item_bytes).decode()
+            else:
+                # old-style, config was stored as JSON string.
+                # Consider deleting this code path in 2025
+                item_json = item["config"]["S"]
+            out = json.loads(item_json)
         return out
 
     def create_request(
@@ -151,7 +161,11 @@ def create_config_request(self, config):
                         "Item": {
                             "from_date": {"S": self.from_date},
                             "config_id": {"S": "exodus-config"},
-                            "config": {"S": json.dumps(config)},
+                            "config": {
+                                "B": b64encode(
+                                    gzip.compress(json.dumps(config).encode())
+                                ).decode()
+                            },
                         }
                     }
                 },

tests/aws/test_dynamodb.py

@@ -145,7 +145,7 @@ def test_batch_write(
     )
 
 
-def test_batch_write_item_limit(fake_publish, caplog):
+def test_batch_write_item_limit(mock_boto3_client, fake_publish, caplog):
     items = fake_publish.items * 9
     ddb = dynamodb.DynamoDB("test", Settings(), NOW_UTC)
 
@@ -218,7 +218,9 @@ def test_write_batch(delete, mock_boto3_client, fake_publish, caplog):
 
 
 @mock.patch("exodus_gw.aws.dynamodb.DynamoDB.batch_write")
-def test_write_batch_put_fail(mock_batch_write, fake_publish, caplog):
+def test_write_batch_put_fail(
+    mock_batch_write, mock_boto3_client, fake_publish, caplog
+):
     caplog.set_level(logging.INFO, logger="exodus-gw")
     mock_batch_write.return_value = {
         "UnprocessedItems": {
@@ -235,7 +237,9 @@ def test_write_batch_put_fail(mock_batch_write, fake_publish, caplog):
 
 
 @mock.patch("exodus_gw.aws.dynamodb.DynamoDB.batch_write")
-def test_write_batch_delete_fail(mock_batch_write, fake_publish, caplog):
+def test_write_batch_delete_fail(
+    mock_batch_write, mock_boto3_client, fake_publish, caplog
+):
     mock_batch_write.return_value = {
         "UnprocessedItems": {
             "my-table": [

tests/conftest.py

@@ -1,4 +1,5 @@
 import base64
+import gzip
 import json
 import os
 from datetime import datetime
@@ -32,8 +33,12 @@ def mock_aws_client():
         yield aws_client
 
 
-@pytest.fixture()
-def fake_dynamodb_query(fake_config: dict[str, Any]):
+@pytest.fixture(params=["binary-config", "text-config"])
+def fake_dynamodb_query(
+    fake_config: dict[str, Any], request: pytest.FixtureRequest
+):
+    binary_config = request.param == "binary-config"
+
     # Returns a callable which can be used as a mock side-effect
     # to make a DynamoDB query on exodus-config return the current
     # fake_config.
@@ -47,22 +52,36 @@ def side_effect(
         # This is the only query we expect right now.
         assert TableName == "my-config"
         assert Limit == 1
+        config_json = json.dumps(fake_config)
+        if binary_config:
+            config_value = {
+                "B": base64.b64encode(
+                    gzip.compress(config_json.encode())
+                ).decode()
+            }
+        else:
+            config_value = {"S": config_json}
         return {
             "Count": 1,
-            "Items": [{"config": {"S": json.dumps(fake_config)}}],
+            "Items": [{"config": config_value}],
         }
 
     return side_effect
 
 
 @pytest.fixture(autouse=True)
-def mock_boto3_client(fake_dynamodb_query):
+def mock_boto3_session():
     with mock.patch("boto3.session.Session") as mock_session:
-        client = mock.MagicMock()
-        client.query.side_effect = fake_dynamodb_query
-        client.__enter__.return_value = client
-        mock_session().client.return_value = client
-        yield client
+        yield mock_session
+
+
+@pytest.fixture()
+def mock_boto3_client(fake_dynamodb_query, mock_boto3_session):
+    client = mock.MagicMock()
+    client.query.side_effect = fake_dynamodb_query
+    client.__enter__.return_value = client
+    mock_boto3_session().client.return_value = client
+    yield client
 
 
 @pytest.fixture()

tests/worker/test_cdn_cache.py

@@ -101,6 +101,7 @@ def test_flush_cdn_cache_fastpurge_disabled(
     db: Session,
     caplog: pytest.LogCaptureFixture,
     fake_message_id: str,
+    mock_boto3_client,
 ):
     """flush_cdn_cache succeeds but does nothing if fastpurge is not configured."""
     settings = load_settings()

tests/worker/test_deploy.py

@@ -1,6 +1,8 @@
+import gzip
 import json
 import logging
 import uuid
+from base64 import b64encode
 from datetime import datetime, timezone
 
 import mock
@@ -52,7 +54,11 @@ def test_deploy_config(
                     "Item": {
                         "from_date": {"S": NOW_UTC},
                         "config_id": {"S": "exodus-config"},
-                        "config": {"S": json.dumps(fake_config)},
+                        "config": {
+                            "B": b64encode(
+                                gzip.compress(json.dumps(fake_config).encode())
+                            ).decode()
+                        },
                     }
                 }
             },
@@ -152,7 +158,13 @@ def test_deploy_config_with_flush(
                     "Item": {
                         "from_date": {"S": NOW_UTC},
                         "config_id": {"S": "exodus-config"},
-                        "config": {"S": json.dumps(updated_config)},
+                        "config": {
+                            "B": b64encode(
+                                gzip.compress(
+                                    json.dumps(updated_config).encode()
+                                )
+                            ).decode()
+                        },
                     }
                 }
             },
@@ -201,7 +213,12 @@ def test_deploy_config_with_flush(
 @mock.patch("exodus_gw.worker.deploy.CurrentMessage.get_current_message")
 @mock.patch("exodus_gw.worker.deploy.DynamoDB.batch_write")
 def test_deploy_config_exception(
-    mock_batch_write, mock_get_message, db, fake_config, caplog
+    mock_batch_write,
+    mock_get_message,
+    db,
+    fake_config,
+    caplog,
+    mock_boto3_client,
 ):
     caplog.set_level(logging.INFO, logger="exodus-gw")
 
@@ -230,7 +247,12 @@ def test_deploy_config_exception(
 @mock.patch("exodus_gw.worker.deploy.CurrentMessage.get_current_message")
 @mock.patch("exodus_gw.worker.deploy.DynamoDB.batch_write")
 def test_deploy_config_bad_state(
-    mock_batch_write, mock_get_message, db, fake_config, caplog
+    mock_batch_write,
+    mock_get_message,
+    db,
+    fake_config,
+    caplog,
+    mock_boto3_client,
 ):
     # Construct task that would be generated by caller.
     t = _task()

tests/worker/test_publish.py

@@ -13,6 +13,8 @@
 from exodus_gw.models.path import PublishedPath
 from exodus_gw.settings import load_settings
 
+pytestmark = pytest.mark.usefixtures("mock_boto3_client")
+
 NOW_UTC = datetime.utcnow()