diff --git a/lambdas/indexer/index.py b/lambdas/indexer/index.py index a315c53197b..bb6a9422229 100644 --- a/lambdas/indexer/index.py +++ b/lambdas/indexer/index.py @@ -503,12 +503,13 @@ def select_package_stats(bucket, manifest_key) -> Optional[dict]: ) payload = resp["Payload"].read() - # FIXME: error handling, return None on errors if "FunctionError" in resp: - raise Exception(payload.decode()) + logger_.error("DuckDB select unhandled error: %s", payload) + return None parsed = json.loads(payload) if "error" in parsed: - raise Exception(parsed["error"]) + logger_.error("DuckDB select error: %s", parsed["error"]) + return None rows = parsed["rows"] return rows[0] if rows else None diff --git a/lambdas/indexer/pytest.ini b/lambdas/indexer/pytest.ini index dd07825516f..f9355a4fbaf 100644 --- a/lambdas/indexer/pytest.ini +++ b/lambdas/indexer/pytest.ini @@ -1,4 +1,6 @@ [pytest] +env = + DUCKDB_SELECT_LAMBDA_ARN = "arn:aws:lambda:us-west-2:123456789012:function:select-lambda" log_cli = True # This is set above critical to prevent logger events from confusing output in CI -log_level = 51 +log_level = 51 diff --git a/lambdas/indexer/test-requirements.txt b/lambdas/indexer/test-requirements.txt index e75e43e319b..b8fc13134ea 100644 --- a/lambdas/indexer/test-requirements.txt +++ b/lambdas/indexer/test-requirements.txt @@ -5,4 +5,5 @@ pluggy==0.9 py==1.10.0 pytest==4.4.0 pytest-cov==2.6.1 +pytest-env==0.6.2 responses==0.10.14 diff --git a/lambdas/indexer/test/test_index.py b/lambdas/indexer/test/test_index.py index c53e3bfa8de..8c45752cc12 100644 --- a/lambdas/indexer/test/test_index.py +++ b/lambdas/indexer/test/test_index.py @@ -979,7 +979,7 @@ def test_index_if_package_select_stats_fail(self, append_mock, select_meta_mock, ) select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) - select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) + select_stats_mock.assert_called_once_with(bucket, manifest_key) append_mock.assert_called_once_with({ "_index": bucket + PACKAGE_INDEX_SUFFIX, "_id": key, @@ -1023,7 +1023,7 @@ def test_index_if_package(self, append_mock, select_meta_mock, select_stats_mock ) select_meta_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) - select_stats_mock.assert_called_once_with(self.s3_client, bucket, manifest_key) + select_stats_mock.assert_called_once_with(bucket, manifest_key) append_mock.assert_called_once_with({ "_index": bucket + PACKAGE_INDEX_SUFFIX, "_id": key, @@ -1182,50 +1182,6 @@ def test_extension_overrides(self): assert self._get_contents('foo.txt', '.txt') == "" assert self._get_contents('foo.ipynb', '.ipynb') == "" - @pytest.mark.xfail( - raises=ParamValidationError, - reason="boto bug https://github.com/boto/botocore/issues/1621", - strict=True, - ) - def test_stub_select_object_content(self): - """Demonstrate that mocking S3 select with boto3 is broken""" - sha_hash = "50f4d0fc2c22a70893a7f356a4929046ce529b53c1ef87e28378d92b884691a5" - manifest_key = f"{MANIFEST_PREFIX_V1}{sha_hash}" - # this SHOULD work, but due to botocore bugs it does not - self.s3_stubber.add_response( - method="select_object_content", - service_response={ - "ResponseMetadata": ANY, - # it is sadly not possible to mock S3 select responses because - # boto incorrectly believes "Payload"'s value should be a dict - # but it's really an iterable in realworld code - # see https://github.com/boto/botocore/issues/1621 - "Payload": [ - { - "Stats": {} - }, - { - "Records": { - "Payload": json.dumps(MANIFEST_DATA).encode(), - }, - }, - { - "End": {} - }, - ] - }, - expected_params={ - "Bucket": "test-bucket", - "Key": manifest_key, - "Expression": index.SELECT_PACKAGE_META, - "ExpressionType": "SQL", - "InputSerialization": { - 'JSON': {'Type': 'LINES'}, - 'CompressionType': 'NONE' - }, - "OutputSerialization": {'JSON': {'RecordDelimiter': '\n'}} - } - ) def test_synthetic_copy_event(self): """check synthetic ObjectCreated:Copy event vs organic obtained on 26-May-2020