Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skip HEAD for multipart check when multipart_threshold is 0 #316

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion s3transfer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,7 @@ def upload_file(
unique_id='s3upload-callback-enable',
)
if (
self._config.multipart_threshold is not None and
self._osutil.get_file_size(filename)
>= self._config.multipart_threshold
):
Expand Down Expand Up @@ -803,7 +804,7 @@ def download_file(
def _download_file(
self, bucket, key, filename, object_size, extra_args, callback
):
if object_size >= self._config.multipart_threshold:
if self._config.multipart_threshold is not None and object_size >= self._config.multipart_threshold:
self._ranged_download(
bucket, key, filename, object_size, extra_args, callback
)
Expand Down
2 changes: 1 addition & 1 deletion s3transfer/copies.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ def _submit(

# If it is greater than threshold do a multipart copy, otherwise
# do a regular copy object.
if transfer_future.meta.size < config.multipart_threshold:
if config.multipart_threshold is None or transfer_future.meta.size < config.multipart_threshold:
self._submit_copy_request(
client, config, osutil, request_executor, transfer_future
)
Expand Down
4 changes: 2 additions & 2 deletions s3transfer/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ def _submit(
:param bandwidth_limiter: The bandwidth limiter to use when
downloading streams
"""
if transfer_future.meta.size is None:
if transfer_future.meta.size is None and config.multipart_threshold is not None:
# If a size was not provided figure out the size for the
# user.
response = client.head_object(
Expand All @@ -364,7 +364,7 @@ def _submit(

# If it is greater than threshold do a ranged download, otherwise
# do a regular GetObject download.
if transfer_future.meta.size < config.multipart_threshold:
if config.multipart_threshold is None or transfer_future.meta.size < config.multipart_threshold:
self._submit_download_request(
client,
config,
Expand Down
2 changes: 1 addition & 1 deletion s3transfer/processpool.py
Original file line number Diff line number Diff line change
Expand Up @@ -816,7 +816,7 @@ def _do_run(self):
def _submit_get_object_jobs(self, download_file_request):
size = self._get_size(download_file_request)
temp_filename = self._allocate_temp_file(download_file_request, size)
if size < self._transfer_config.multipart_threshold:
if self._transfer_config.multiparth_threshold is None or size < self._transfer_config.multipart_threshold:
self._submit_single_get_object_job(
download_file_request, temp_filename
)
Expand Down
6 changes: 3 additions & 3 deletions s3transfer/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def provide_transfer_size(self, transfer_future):
)

def requires_multipart_upload(self, transfer_future, config):
return transfer_future.meta.size >= config.multipart_threshold
return config.multipart_threshold is not None and transfer_future.meta.size >= config.multipart_threshold

def get_put_object_body(self, transfer_future):
# Get a file-like object for the given input
Expand Down Expand Up @@ -393,14 +393,14 @@ def provide_transfer_size(self, transfer_future):
def requires_multipart_upload(self, transfer_future, config):
# If the user has set the size, we can use that.
if transfer_future.meta.size is not None:
return transfer_future.meta.size >= config.multipart_threshold
return config.multipart_threshold is not None and transfer_future.meta.size >= config.multipart_threshold

# This is tricky to determine in this case because we can't know how
# large the input is. So to figure it out, we read data into memory
# up until the threshold and compare how much data was actually read
# against the threshold.
fileobj = transfer_future.meta.call_args.fileobj
threshold = config.multipart_threshold
threshold = config.multipart_threshold or 8 * 1024 * 1024
self._initial_data = self._read(fileobj, threshold, False)
if len(self._initial_data) < threshold:
return False
Expand Down