fix tets

lambdas/indexer/index.py

@@ -503,12 +503,13 @@ def select_package_stats(bucket, manifest_key) -> Optional[dict]:
     )
 
     payload = resp["Payload"].read()
-    # FIXME: error handling, return None on errors
     if "FunctionError" in resp:
-        raise Exception(payload.decode())
+        logger_.error("DuckDB select unhandled error: %s", payload)
+        return None
     parsed = json.loads(payload)
     if "error" in parsed:
-        raise Exception(parsed["error"])
+        logger_.error("DuckDB select error: %s", parsed["error"])
+        return None
 
     rows = parsed["rows"]
     return rows[0] if rows else None

lambdas/indexer/pytest.ini

@@ -1,4 +1,6 @@
 [pytest]
+env =
+    DUCKDB_SELECT_LAMBDA_ARN = "arn:aws:lambda:us-west-2:123456789012:function:select-lambda"
 log_cli = True
 # This is set above critical to prevent logger events from confusing output in CI
-log_level = 51 
+log_level = 51

lambdas/indexer/test-requirements.txt

@@ -5,4 +5,5 @@ pluggy==0.9
 py==1.10.0
 pytest==4.4.0
 pytest-cov==2.6.1
+pytest-env==0.6.2
 responses==0.10.14

lambdas/indexer/test/test_index.py

@@ -979,7 +979,7 @@ def test_index_if_package_select_stats_fail(self, append_mock, select_meta_mock,
         )
 
         select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key)
-        select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key)
+        select_stats_mock.assert_called_once_with(bucket, manifest_key)
         append_mock.assert_called_once_with({
             "_index": bucket + PACKAGE_INDEX_SUFFIX,
             "_id": key,
@@ -1023,7 +1023,7 @@ def test_index_if_package(self, append_mock, select_meta_mock, select_stats_mock
         )
 
         select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key)
-        select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key)
+        select_stats_mock.assert_called_once_with(bucket, manifest_key)
         append_mock.assert_called_once_with({
             "_index": bucket + PACKAGE_INDEX_SUFFIX,
             "_id": key,
@@ -1182,50 +1182,6 @@ def test_extension_overrides(self):
             assert self._get_contents('foo.txt', '.txt') == ""
             assert self._get_contents('foo.ipynb', '.ipynb') == ""
 
-    @pytest.mark.xfail(
-        raises=ParamValidationError,
-        reason="boto bug https://github.com/boto/botocore/issues/1621",
-        strict=True,
-    )
-    def test_stub_select_object_content(self):
-        """Demonstrate that mocking S3 select with boto3 is broken"""
-        sha_hash = "50f4d0fc2c22a70893a7f356a4929046ce529b53c1ef87e28378d92b884691a5"
-        manifest_key = f"{MANIFEST_PREFIX_V1}{sha_hash}"
-        # this SHOULD work, but due to botocore bugs it does not
-        self.s3_stubber.add_response(
-            method="select_object_content",
-            service_response={
-                "ResponseMetadata": ANY,
-                # it is sadly not possible to mock S3 select responses because
-                # boto incorrectly believes "Payload"'s value should be a dict
-                # but it's really an iterable in realworld code
-                # see https://github.com/boto/botocore/issues/1621
-                "Payload": [
-                    {
-                        "Stats": {}
-                    },
-                    {
-                        "Records": {
-                            "Payload": json.dumps(MANIFEST_DATA).encode(),
-                        },
-                    },
-                    {
-                        "End": {}
-                    },
-                ]
-            },
-            expected_params={
-                "Bucket": "test-bucket",
-                "Key": manifest_key,
-                "Expression": index.SELECT_PACKAGE_META,
-                "ExpressionType": "SQL",
-                "InputSerialization": {
-                    'JSON': {'Type': 'LINES'},
-                    'CompressionType': 'NONE'
-                },
-                "OutputSerialization": {'JSON': {'RecordDelimiter': '\n'}}
-            }
-        )
 
     def test_synthetic_copy_event(self):
         """check synthetic ObjectCreated:Copy event vs organic obtained on 26-May-2020