Skip to content

Commit

Permalink
Fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
ddelange committed Dec 15, 2024
1 parent 1266ddb commit 2e9e317
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 15 deletions.
10 changes: 10 additions & 0 deletions .github/workflows/python-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
- {python-version: '3.10', os: windows-2019}
- {python-version: '3.11', os: windows-2019}
- {python-version: '3.12', os: windows-2019}
- {python-version: '3.13', os: windows-2019}
steps:
- uses: actions/checkout@v2

Expand Down Expand Up @@ -78,6 +79,9 @@ jobs:
# - {python-version: '3.8', os: windows-2019}
# - {python-version: '3.9', os: windows-2019}
# - {python-version: '3.10', os: windows-2019}
# - {python-version: '3.11', os: windows-2019}
# - {python-version: '3.12', os: windows-2019}
# - {python-version: '3.13', os: windows-2019}

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -114,6 +118,9 @@ jobs:
# - {python-version: '3.8', os: windows-2019}
# - {python-version: '3.9', os: windows-2019}
# - {python-version: '3.10', os: windows-2019}
# - {python-version: '3.11', os: windows-2019}
# - {python-version: '3.12', os: windows-2019}
# - {python-version: '3.13', os: windows-2019}

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -162,6 +169,9 @@ jobs:
# - {python-version: '3.8', os: windows-2019}
# - {python-version: '3.9', os: windows-2019}
# - {python-version: '3.10', os: windows-2019}
# - {python-version: '3.11', os: windows-2019}
# - {python-version: '3.12', os: windows-2019}
# - {python-version: '3.13', os: windows-2019}

steps:
- uses: actions/checkout@v2
Expand Down
34 changes: 19 additions & 15 deletions smart_open/azure.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ def __init__(
max_concurrency=DEFAULT_MAX_CONCURRENCY,
):
self._container_name = container
self._blob_name = blob

self._blob = _get_blob_client(client, container, blob)
# type: azure.storage.blob.BlobClient
Expand Down Expand Up @@ -369,20 +370,25 @@ def __exit__(self, exc_type, exc_val, exc_tb):
self.close()

def __str__(self):
return "(%s, %r, %r)" % (self.__class__.__name__,
self._container_name,
self._blob.blob_name)
return "(%s, %r, %r)" % (
self.__class__.__name__,
self._container_name,
self._blob_name
)

def __repr__(self):
return "%s(container=%r, blob=%r)" % (
self.__class__.__name__, self._container_name, self._blob.blob_name,
self.__class__.__name__,
self._container_name,
self._blob_name,
)


class Writer(io.BufferedIOBase):
"""Writes bytes to Azure Blob Storage.
Implements the io.BufferedIOBase interface of the standard library."""
_blob = None # always initialized so closed property is functional in case _get_blob_client fails

def __init__(
self,
Expand All @@ -392,21 +398,19 @@ def __init__(
blob_kwargs=None,
min_part_size=_DEFAULT_MIN_PART_SIZE,
):
self._is_closed = False
self._container_name = container

self._blob = _get_blob_client(client, container, blob)
self._blob_name = blob
self._blob_kwargs = blob_kwargs or {}
# type: azure.storage.blob.BlobClient

self._min_part_size = min_part_size

self._total_size = 0
self._total_parts = 0
self._bytes_uploaded = 0
self._current_part = io.BytesIO()
self._block_list = []

# type: azure.storage.blob.BlobClient
self._blob = _get_blob_client(client, container, blob)

#
# This member is part of the io.BufferedIOBase interface.
#
Expand All @@ -424,7 +428,7 @@ def terminate(self):
logger.debug('%s: terminating multipart upload', self)
if not self.closed:
self._block_list = []
self._is_closed = True
self._blob = None
logger.debug('%s: terminated multipart upload', self)

#
Expand All @@ -437,12 +441,12 @@ def close(self):
self._upload_part()
self._blob.commit_block_list(self._block_list, **self._blob_kwargs)
self._block_list = []
self._is_closed = True
self._blob = None
logger.debug('%s: completed multipart upload', self)

@property
def closed(self):
return self._is_closed
return self._blob is None

def writable(self):
"""Return True if the stream supports writing."""
Expand Down Expand Up @@ -528,13 +532,13 @@ def __str__(self):
return "(%s, %r, %r)" % (
self.__class__.__name__,
self._container_name,
self._blob.blob_name
self._blob_name
)

def __repr__(self):
return "%s(container=%r, blob=%r, min_part_size=%r)" % (
self.__class__.__name__,
self._container_name,
self._blob.blob_name,
self._blob_name,
self._min_part_size
)
13 changes: 13 additions & 0 deletions smart_open/hdfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,18 @@ def __init__(self, uri):
#
self.raw = None

@property
def closed(self):
return self._sub is None

#
# Override some methods from io.IOBase.
#
def close(self):
"""Flush and close this stream."""
logger.debug("close: called")
if self.closed:
return
self._sub.terminate()
self._sub = None

Expand Down Expand Up @@ -135,10 +141,17 @@ def __init__(self, uri):
#
self.raw = None

@property
def closed(self):
return self._sub is None

def close(self):
if self.closed:
return
self.flush()
self._sub.stdin.close()
self._sub.wait()
self._sub = None

def flush(self):
self._sub.stdin.flush()
Expand Down

0 comments on commit 2e9e317

Please sign in to comment.