From b0959b3e0f62e3393a2848ea2e3c65b9407f15c1 Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:38:19 +0000 Subject: [PATCH] Resolving int test bugs --- poetry.lock | 28 ++++++++++++++++------------ pyproject.toml | 2 +- run_integration_tests.py | 1 - tests/integration/test_tap_s3_csv.py | 20 +++++++++++++------- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index db09637..95093d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -132,17 +132,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.34.158" +version = "1.35.8" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.158-py3-none-any.whl", hash = "sha256:c29e9b7e1034e8734ccaffb9f2b3f3df2268022fd8a93d836604019f8759ce27"}, - {file = "boto3-1.34.158.tar.gz", hash = "sha256:5b7b2ce0ec1e498933f600d29f3e1c641f8c44dd7e468c26795359d23d81fa39"}, + {file = "boto3-1.35.8-py3-none-any.whl", hash = "sha256:06eac4757de2a9c6020381205cb902f05964caad80b56e58c8931284a133b4cb"}, + {file = "boto3-1.35.8.tar.gz", hash = "sha256:b9587131372a808bf6f99c5ed8b11be55cd113261cc3b437a917b4acc6c30bfe"}, ] [package.dependencies] -botocore = ">=1.34.158,<1.35.0" +botocore = ">=1.35.8,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -151,13 +151,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.162" +version = "1.35.8" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, - {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, + {file = "botocore-1.35.8-py3-none-any.whl", hash = "sha256:adf389eb8fd87775f193300e3431d1353f925807ad3a39958172cb644f0d60a1"}, + {file = "botocore-1.35.8.tar.gz", hash = "sha256:4b820cf680ab5d778bd2fe4feeef1ff8a2b96d5c535d4638ab30f703ade282f8"}, ] [package.dependencies] @@ -1746,20 +1746,24 @@ files = [ [[package]] name = "zipp" -version = "3.20.0" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, - {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.13" -content-hash = "e22f8f5375aa7229e732606cbec0ad4f59f95ff5cbd37499f09dd554e8081fec" +content-hash = "7f276dc8a1d21a6b88d23a9711a760228eab0d0f9304f4d9452bec262ca7a2fb" diff --git a/pyproject.toml b/pyproject.toml index 0b832a3..2a84081 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ include = [ [tool.poetry.dependencies] python = ">=3.8,<3.13" realit_singer_python = "^5.0.0" -boto3 = "1.34.158" +boto3 = "1.35.8" realit-singer-encodings = ">=2.1.0" voluptuous = "0.14.2" ujson = ">=5.4.0" diff --git a/run_integration_tests.py b/run_integration_tests.py index c56e407..d5abe97 100644 --- a/run_integration_tests.py +++ b/run_integration_tests.py @@ -45,7 +45,6 @@ def run_command(command, env=None): container_name = "minio_server" result = is_container_running(container_name) print(result) - print(os.environ) if result or 'GITHUB_ACTIONS' in os.environ: rc = run_command(['poetry', 'run', 'pytest', 'tests/integration'],env=os.environ) raise SystemExit(rc) diff --git a/tests/integration/test_tap_s3_csv.py b/tests/integration/test_tap_s3_csv.py index 04ac215..72bd30d 100644 --- a/tests/integration/test_tap_s3_csv.py +++ b/tests/integration/test_tap_s3_csv.py @@ -5,6 +5,7 @@ import os.path import random import unittest +import sys from copy import deepcopy import boto3 @@ -149,15 +150,18 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: - s3_client = boto3.client("s3") + # Remove test file from bucket + s3_client = boto3.client( + "s3", endpoint_url=cls.config["aws_endpoint_url"] + ) s3_client.delete_object(Bucket=cls.config["bucket"], Key=cls.obj_name) def test_discovery(self): - f = io.StringIO() - with contextlib.redirect_stdout(f): + file_buffer = io.StringIO() + with contextlib.redirect_stdout(file_buffer): do_discover(self.config) - catalog = ujson.loads(f.getvalue()) + catalog = ujson.loads(file_buffer.getvalue()) self.assertIsInstance(catalog, dict) self.assertEqual(1, len(catalog["streams"])) @@ -188,12 +192,14 @@ def test_sync(self): # set stream to selected catalog["streams"][0]["metadata"][0]["metadata"]["selected"] = True - f = io.StringIO() - with contextlib.redirect_stdout(f): + file_buffer = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) + with contextlib.redirect_stdout(file_buffer): do_sync(self.config, catalog, {}) + # Position at the start of the buffer + file_buffer.seek(0) lines = [ - ujson.loads(line) for line in f.getvalue().strip().splitlines() + ujson.loads(line) for line in file_buffer.read().strip().splitlines() ] self.assertDictEqual({"type": "STATE", "value": {}}, lines[0])