From c67b03c7d88533773d62d72f0b626031805d61eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Socha?= <31014760+lukaszsocha2@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:03:35 +0200 Subject: [PATCH] feat: Use upload session `urls` for chunk upload (#875) Closes: SDK-3836 --- boxsdk/object/file.py | 25 +++- boxsdk/object/folder.py | 23 ++- boxsdk/object/upload_session.py | 28 +++- docs/usage/configuration.md | 2 + docs/usage/files.md | 50 ++++--- setup.py | 3 +- test/integration_new/object/folder_itest.py | 19 ++- test/unit/object/test_file.py | 18 ++- test/unit/object/test_folder.py | 76 ++++++---- test/unit/object/test_upload_session.py | 157 +++++++++++++++++--- 10 files changed, 303 insertions(+), 98 deletions(-) diff --git a/boxsdk/object/file.py b/boxsdk/object/file.py index b63c6662..c9128a05 100644 --- a/boxsdk/object/file.py +++ b/boxsdk/object/file.py @@ -49,7 +49,9 @@ def preflight_check(self, size: int, name: Optional[str] = None) -> Optional[str ) @api_call - def create_upload_session(self, file_size: int, file_name: Optional[str] = None) -> 'UploadSession': + def create_upload_session( + self, file_size: int, file_name: Optional[str] = None, use_upload_session_urls: bool = True + ) -> 'UploadSession': """ Create a new chunked upload session for uploading a new version of the file. @@ -57,6 +59,11 @@ def create_upload_session(self, file_size: int, file_name: Optional[str] = None) The size of the file in bytes that will be uploaded. :param file_name: The new name of the file version that will be uploaded. + :param use_upload_session_urls: + The parameter detrermining what urls to use to perform chunked upload. + If True, the urls returned by create_upload_session() endpoint response will be used, + unless a custom API.UPLOAD_URL was set in the config. + If False, the base upload url will be used. :returns: A :class:`UploadSession` object. """ @@ -68,13 +75,18 @@ def create_upload_session(self, file_size: int, file_name: Optional[str] = None) body_params['file_name'] = file_name url = self.get_url('upload_sessions').replace(self.session.api_config.BASE_API_URL, self.session.api_config.UPLOAD_URL) response = self._session.post(url, data=json.dumps(body_params)).json() - return self.translator.translate( + upload_session = self.translator.translate( session=self._session, response_object=response, ) + # pylint:disable=protected-access + upload_session._use_upload_session_urls = use_upload_session_urls + return upload_session @api_call - def get_chunked_uploader(self, file_path: str, rename_file: bool = False) -> 'ChunkedUploader': + def get_chunked_uploader( + self, file_path: str, rename_file: bool = False, use_upload_session_urls: bool = True + ) -> 'ChunkedUploader': # pylint: disable=consider-using-with """ Instantiate the chunked upload instance and create upload session with path to file. @@ -83,13 +95,18 @@ def get_chunked_uploader(self, file_path: str, rename_file: bool = False) -> 'Ch The local path to the file you wish to upload. :param rename_file: Indicates whether the file should be renamed or not. + :param use_upload_session_urls: + The parameter detrermining what urls to use to perform chunked upload. + If True, the urls returned by create_upload_session() endpoint response will be used, + unless a custom API.UPLOAD_URL was set in the config. + If False, the base upload url will be used. :returns: A :class:`ChunkedUploader` object. """ total_size = os.stat(file_path).st_size content_stream = open(file_path, 'rb') file_name = os.path.basename(file_path) if rename_file else None - upload_session = self.create_upload_session(total_size, file_name) + upload_session = self.create_upload_session(total_size, file_name, use_upload_session_urls) return upload_session.get_chunked_uploader_for_stream(content_stream, total_size) def _get_accelerator_upload_url_for_update(self) -> Optional[str]: diff --git a/boxsdk/object/folder.py b/boxsdk/object/folder.py index 20537ebd..037ae575 100644 --- a/boxsdk/object/folder.py +++ b/boxsdk/object/folder.py @@ -115,7 +115,7 @@ def preflight_check(self, size: int, name: str) -> Optional[str]: ) @api_call - def create_upload_session(self, file_size: int, file_name: str) -> 'UploadSession': + def create_upload_session(self, file_size: int, file_name: str, use_upload_session_urls: bool = True) -> 'UploadSession': """ Creates a new chunked upload session for upload a new file. @@ -123,6 +123,11 @@ def create_upload_session(self, file_size: int, file_name: str) -> 'UploadSessio The size of the file in bytes that will be uploaded. :param file_name: The name of the file that will be uploaded. + :param use_upload_session_urls: + The parameter detrermining what urls to use to perform chunked upload. + If True, the urls returned by create_upload_session() endpoint response will be used, + unless a custom API.UPLOAD_URL was set in the config. + If False, the base upload url will be used. :returns: A :class:`UploadSession` object. """ @@ -133,13 +138,18 @@ def create_upload_session(self, file_size: int, file_name: str) -> 'UploadSessio 'file_name': file_name, } response = self._session.post(url, data=json.dumps(body_params)).json() - return self.translator.translate( + upload_session = self.translator.translate( session=self._session, response_object=response, ) + # pylint:disable=protected-access + upload_session._use_upload_session_urls = use_upload_session_urls + return upload_session @api_call - def get_chunked_uploader(self, file_path: str, file_name: Optional[str] = None) -> 'ChunkedUploader': + def get_chunked_uploader( + self, file_path: str, file_name: Optional[str] = None, use_upload_session_urls: bool = True + ) -> 'ChunkedUploader': # pylint: disable=consider-using-with """ Instantiate the chunked upload instance and create upload session with path to file. @@ -149,6 +159,11 @@ def get_chunked_uploader(self, file_path: str, file_name: Optional[str] = None) :param file_name: The name with extention of the file that will be uploaded, e.g. new_file_name.zip. If not specified, the name from the local system is used. + :param use_upload_session_urls: + The parameter detrermining what urls to use to perform chunked upload. + If True, the urls returned by create_upload_session() endpoint response will be used, + unless a custom API.UPLOAD_URL was set in the config. + If False, the base upload url will be used. :returns: A :class:`ChunkedUploader` object. """ @@ -157,7 +172,7 @@ def get_chunked_uploader(self, file_path: str, file_name: Optional[str] = None) content_stream = open(file_path, 'rb') try: - upload_session = self.create_upload_session(total_size, upload_file_name) + upload_session = self.create_upload_session(total_size, upload_file_name, use_upload_session_urls) return upload_session.get_chunked_uploader_for_stream(content_stream, total_size) except Exception: content_stream.close() diff --git a/boxsdk/object/upload_session.py b/boxsdk/object/upload_session.py index 0d54f12e..8ccbe547 100644 --- a/boxsdk/object/upload_session.py +++ b/boxsdk/object/upload_session.py @@ -8,6 +8,8 @@ from boxsdk import BoxAPIException from boxsdk.util.api_call_decorator import api_call from boxsdk.util.chunked_uploader import ChunkedUploader +from boxsdk.session.session import Session +from boxsdk.config import API from .base_object import BaseObject from ..pagination.limit_offset_based_dict_collection import LimitOffsetBasedDictCollection @@ -19,11 +21,22 @@ class UploadSession(BaseObject): _item_type = 'upload_session' _parent_item_type = 'file' + _default_upload_url = API.UPLOAD_URL - def get_url(self, *args: Any) -> str: + def __init__( + self, session: Session, object_id: str, response_object: dict = None, use_upload_session_urls: bool = True + ): + super().__init__(session, object_id, response_object) + self._use_upload_session_urls = use_upload_session_urls + + def get_url(self, *args: Any, url_key: str = None) -> str: """ Base class override. Endpoint is a little different - it's /files/upload_sessions. """ + session_endpoints = getattr(self, 'session_endpoints', {}) + if self._use_upload_session_urls and url_key in session_endpoints and self.session.api_config.UPLOAD_URL == self._default_upload_url: + return session_endpoints[url_key] + return self._session.get_url( f'{self._parent_item_type}s/{self._item_type}s', self._object_id, @@ -44,7 +57,7 @@ def get_parts(self, limit: Optional[int] = None, offset: Optional[int] = None) - """ return LimitOffsetBasedDictCollection( session=self.session, - url=self.get_url('parts'), + url=self.get_url('parts', url_key='list_parts'), limit=limit, offset=offset, fields=None, @@ -87,7 +100,7 @@ def upload_part_bytes( 'Content-Range': f'bytes {offset}-{range_end}/{total_size}', } response = self._session.put( - self.get_url(), + self.get_url(url_key='upload_part'), headers=headers, data=part_bytes, ) @@ -131,7 +144,7 @@ def commit( try: response = self._session.post( - self.get_url('commit'), + self.get_url('commit', url_key='commit'), headers=headers, data=json.dumps(body), ) @@ -154,7 +167,12 @@ def abort(self) -> bool: :returns: A boolean indication success of the upload abort. """ - return self.delete() + + box_response = self._session.delete( + self.get_url(url_key='abort'), + expect_json_response=False + ) + return box_response.ok def get_chunked_uploader_for_stream(self, content_stream: IO[bytes], file_size: int) -> ChunkedUploader: """ diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 612da4e1..c12848d1 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -64,6 +64,8 @@ API.OAUTH2_AUTHORIZE_URL = 'https://my-company.com/authorize' ### Upload URL The default URL used when uploading files to Box can be changed by assigning a new value to the `API.UPLOAD_URL` field. +If this variable is ever changed from default value, the SDK will alwayse use this URL to upload files to Box, +even if `use_upload_session_urls` is set to `True` while creating an upload session for a chunked upload. ```python from boxsdk.config import API diff --git a/docs/usage/files.md b/docs/usage/files.md index 8cbc44e2..125b2abb 100644 --- a/docs/usage/files.md +++ b/docs/usage/files.md @@ -194,9 +194,14 @@ Chunked Upload -------------- For large files or in cases where the network connection is less reliable, -you may want to upload the file in parts. This allows a single part to fail +you may want to upload the file in parts. This allows a single part to fail without aborting the entire upload, and failed parts can then be retried. +Since box-python-sdk 3.11.0 release, by default the SDK uses upload urls provided in response +when creating a new upload session. This allowes to always upload your content to the closest Box data center and +can significantly improve upload speed. You can always disable this feature and always use base upload url by +setting `use_upload_session_urls` flag to `False` when creating upload session. + ### Automatic Uploader Since box-python-sdk 3.7.0 release, automatic uploader uses multiple threads, which significantly speeds up the upload process. @@ -211,9 +216,11 @@ API.CHUNK_UPLOAD_THREADS = 6 #### Upload new file The SDK provides a method of automatically handling a chunked upload. First get a folder you want to upload the file to. -Then call [`folder.get_chunked_uploader(file_path, rename_file=False)`][get_chunked_uploader_for_file] to retrieve -a [`ChunkedUploader`][chunked_uploader_class] object. Calling the method [`chunked_upload.start()`][start] will -kick off the chunked upload process and return the [File][file_class] +Then call [`folder.get_chunked_uploader(file_path, rename_file=False, use_upload_session_urls=True)`][get_chunked_uploader_for_file] +to retrieve a [`ChunkedUploader`][chunked_uploader_class] object. Setting `use_upload_session_urls` to `True` inilializes +the uploader that utlizies urls returned by the `Create Upload Session` endpoint response unless a custom +API.UPLOAD_URL was set in the config. Setting `use_upload_session_urls` to `False` inilializes the uploader that uses always base upload urls. +Calling the method [`chunked_upload.start()`][start] will kick off the chunked upload process and return the [File][file_class] object that was uploaded. @@ -224,7 +231,10 @@ uploaded_file = chunked_uploader.start() print(f'File "{uploaded_file.name}" uploaded to Box with file ID {uploaded_file.id}') ``` -You can also upload file stream by creating a [`UploadSession`][upload_session_class] first and then calling the +You can also upload file stream by creating a [`UploadSession`][upload_session_class] first. This can be done by calling +[`folder.create_upload_session(file_size, file_name=None, use_upload_session_urls=True)`][create_upload_session] method. +`use_upload_session_urls` flag is used to determine if the upload session should use urls returned by +the `Create Upload Session` endpoint or should it always use base upload urls. Then you can call method [`upload_session.get_chunked_uploader_for_stream(content_stream, file_size)`][get_chunked_uploader_for_stream]. ```python @@ -240,14 +250,14 @@ with open(test_file_path, 'rb') as content_stream: #### Upload new file version To upload a new file version for a large file, first get a file you want to replace. -Then call [`file.get_chunked_uploader(file_path)`][get_chunked_uploader_for_version] +Then call [`file.get_chunked_uploader(file_path, rename_file=False, use_upload_session_urls=True)`][get_chunked_uploader_for_version] to retrieve a [`ChunkedUploader`][chunked_uploader_class] object. Calling the method [`chunked_upload.start()`][start] will kick off the chunked upload process and return the updated [File][file_class]. ```python # uploads new large file version -chunked_uploader = client.file('existing_big_file_id').get_chunked_uploader('/path/to/file') +chunked_uploader = client.file('existing_big_file_id').get_chunked_uploader(file_path='/path/to/file') uploaded_file = chunked_uploader.start() print(f'File "{uploaded_file.name}" uploaded to Box with file ID {uploaded_file.id}') # the uploaded_file.id will be the same as 'existing_big_file_id' @@ -293,17 +303,6 @@ except: print(f'File "{uploaded_file.name}" uploaded to Box with file ID {uploaded_file.id}') ``` -Alternatively, you can also create a [`UploadSession`][upload_session_class] object by calling -[`client.upload_session(session_id)`][upload_session] if you have the upload session id. This can be helpful in -resuming an existing upload session. - - -```python -chunked_uploader = client.upload_session('12345').get_chunked_uploader('/path/to/file') -uploaded_file = chunked_uploader.resume() -print(f'File "{uploaded_file.name}" uploaded to Box with file ID {uploaded_file.id}') -``` - [resume]: https://box-python-sdk.readthedocs.io/en/latest/boxsdk.object.html#boxsdk.object.chunked_uploader.ChunkedUploader.resume #### Abort Chunked Upload @@ -317,7 +316,7 @@ from boxsdk.exception import BoxNetworkException test_file_path = '/path/to/large_file.mp4' content_stream = open(test_file_path, 'rb') total_size = os.stat(test_file_path).st_size -chunked_uploader = client.upload_session('56781').get_chunked_uploader_for_stream(content_stream, total_size) +chunked_uploader = client.file('existing_big_file_id').get_chunked_uploader(file_path='/path/to/file') try: uploaded_file = chunked_uploader.start() except BoxNetworkException: @@ -371,8 +370,10 @@ The individual endpoint methods are detailed below: #### Create Upload Session for File Version To create an upload session for uploading a large version, call -[`file.create_upload_session(file_size, file_name=None)`][create_version_upload_session] with the size of the file to be -uploaded. You can optionally specify a new `file_name` to rename the file on upload. This method returns an +[`file.create_upload_session(file_size, file_name=None, use_upload_session_urls=True)`][create_version_upload_session] +with the size of the file to be uploaded. You can optionally specify a new `file_name` to rename the file on upload. +`use_upload_session_urls` flag is used to determine if the upload session should use urls returned by +the `Create Upload Session` endpoint or should it always use base upload urls. This method returns an [`UploadSession`][upload_session_class] object representing the created upload session. @@ -388,9 +389,10 @@ print(f'Created upload session {upload_session.id} with chunk size of {upload_se #### Create Upload Session for File To create an upload session for uploading a new large file, call -[`folder.create_upload_session(file_size, file_name)`][create_upload_session] with the size and filename of the file -to be uploaded. This method returns an [`UploadSession`][upload_session_class] object representing the created upload -session. +[`folder.create_upload_session(file_size, file_name, use_upload_session_urls=True)`][create_upload_session] with +the size and filename of the file to be uploaded. `use_upload_session_urls` flag is used to determine if the upload +session should use urls returned by the `Create Upload Session` endpoint or should it always use base upload urls. +This method returns an [`UploadSession`][upload_session_class] object representing the created upload session. ```python diff --git a/setup.py b/setup.py index 5aa00f52..8c9dba0d 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 6 - Mature', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', @@ -18,6 +18,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Operating System :: OS Independent', diff --git a/test/integration_new/object/folder_itest.py b/test/integration_new/object/folder_itest.py index aad28a15..016ac848 100644 --- a/test/integration_new/object/folder_itest.py +++ b/test/integration_new/object/folder_itest.py @@ -72,9 +72,24 @@ def test_manual_chunked_upload(parent_folder, large_file, large_file_name): util.permanently_delete(uploaded_file) -def test_auto_chunked_upload(parent_folder, large_file, large_file_name): +def test_auto_chunked_upload_using_upload_session_urls(parent_folder, large_file, large_file_name): total_size = os.stat(large_file.path).st_size - chunked_uploader = parent_folder.get_chunked_uploader(large_file.path) + chunked_uploader = parent_folder.get_chunked_uploader(large_file.path, use_upload_session_urls=True) + + uploaded_file = chunked_uploader.start() + + try: + assert uploaded_file.id + assert uploaded_file.name == large_file_name + assert uploaded_file.parent == parent_folder + assert uploaded_file.size == total_size + finally: + util.permanently_delete(uploaded_file) + + +def test_auto_chunked_upload_NOT_using_upload_session_urls(parent_folder, large_file, large_file_name): + total_size = os.stat(large_file.path).st_size + chunked_uploader = parent_folder.get_chunked_uploader(large_file.path, use_upload_session_urls=False) uploaded_file = chunked_uploader.start() diff --git a/test/unit/object/test_file.py b/test/unit/object/test_file.py index e78393b4..21156713 100644 --- a/test/unit/object/test_file.py +++ b/test/unit/object/test_file.py @@ -50,7 +50,8 @@ def test_delete_file(test_file, mock_box_session, etag, if_match_header): ) -def test_create_upload_session(test_file, mock_box_session): +@pytest.mark.parametrize('use_upload_session_urls', [True, False]) +def test_create_upload_session(test_file, mock_box_session, use_upload_session_urls): expected_url = f'{API.UPLOAD_URL}/files/{test_file.object_id}/upload_sessions' file_size = 197520 part_size = 12345 @@ -71,7 +72,9 @@ def test_create_upload_session(test_file, mock_box_session): 'total_parts': total_parts, 'part_size': part_size, } - upload_session = test_file.create_upload_session(file_size, file_name) + upload_session = test_file.create_upload_session( + file_size, file_name, use_upload_session_urls=use_upload_session_urls + ) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) assert isinstance(upload_session, UploadSession) assert upload_session._session == mock_box_session @@ -80,9 +83,13 @@ def test_create_upload_session(test_file, mock_box_session): assert upload_session.num_parts_processed == num_parts_processed assert upload_session.type == upload_session_type assert upload_session.id == upload_session_id + assert upload_session._use_upload_session_urls == use_upload_session_urls -def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file_path, test_file): +@pytest.mark.parametrize('use_upload_session_urls', [True, False]) +def test_get_chunked_uploader( + mock_box_session, mock_content_response, mock_file_path, test_file, use_upload_session_urls +): expected_url = f'{API.UPLOAD_URL}/files/{test_file.object_id}/upload_sessions' mock_file_stream = BytesIO(mock_content_response.content) file_size = 197520 @@ -105,7 +112,9 @@ def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file with patch('os.stat') as stat: stat.return_value.st_size = file_size with patch('boxsdk.object.file.open', return_value=mock_file_stream): - chunked_uploader = test_file.get_chunked_uploader(mock_file_path) + chunked_uploader = test_file.get_chunked_uploader( + mock_file_path, use_upload_session_urls=use_upload_session_urls + ) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) upload_session = chunked_uploader._upload_session assert upload_session.part_size == part_size @@ -113,6 +122,7 @@ def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file assert upload_session.num_parts_processed == num_parts_processed assert upload_session.type == upload_session_type assert upload_session.id == upload_session_id + assert upload_session._use_upload_session_urls == use_upload_session_urls assert isinstance(chunked_uploader, ChunkedUploader) diff --git a/test/unit/object/test_folder.py b/test/unit/object/test_folder.py index 21b099ed..acf27841 100644 --- a/test/unit/object/test_folder.py +++ b/test/unit/object/test_folder.py @@ -77,7 +77,10 @@ def get_response(limit, offset): return get_response -def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file_path, test_folder): +@pytest.mark.parametrize('use_upload_session_urls', [True, False]) +def test_get_chunked_uploader( + mock_box_session, mock_content_response, mock_file_path, test_folder, use_upload_session_urls +): expected_url = f'{API.UPLOAD_URL}/files/upload_sessions' mock_file_stream = BytesIO(mock_content_response.content) file_size = 197520 @@ -102,7 +105,9 @@ def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file with patch('os.stat') as stat: stat.return_value.st_size = file_size with patch('boxsdk.object.folder.open', return_value=mock_file_stream): - chunked_uploader = test_folder.get_chunked_uploader(mock_file_path) + chunked_uploader = test_folder.get_chunked_uploader( + mock_file_path, use_upload_session_urls=use_upload_session_urls + ) mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) upload_session = chunked_uploader._upload_session assert upload_session.part_size == part_size @@ -110,9 +115,45 @@ def test_get_chunked_uploader(mock_box_session, mock_content_response, mock_file assert upload_session.num_parts_processed == num_parts_processed assert upload_session.type == upload_session_type assert upload_session.id == upload_session_id + assert upload_session._use_upload_session_urls is use_upload_session_urls assert isinstance(chunked_uploader, ChunkedUploader) +@pytest.mark.parametrize('use_upload_session_urls', [True, False]) +def test_create_upload_session(test_folder, mock_box_session, use_upload_session_urls): + expected_url = f'{API.UPLOAD_URL}/files/upload_sessions' + file_size = 197520 + file_name = 'test_file.pdf' + upload_session_id = 'F971964745A5CD0C001BBE4E58196BFD' + upload_session_type = 'upload_session' + num_parts_processed = 0 + total_parts = 16 + part_size = 12345 + expected_data = { + 'folder_id': test_folder.object_id, + 'file_size': file_size, + 'file_name': file_name, + } + mock_box_session.post.return_value.json.return_value = { + 'id': upload_session_id, + 'type': upload_session_type, + 'num_parts_processed': num_parts_processed, + 'total_parts': total_parts, + 'part_size': part_size, + } + upload_session = test_folder.create_upload_session( + file_size, file_name, use_upload_session_urls=use_upload_session_urls + ) + mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) + assert isinstance(upload_session, UploadSession) + assert upload_session.part_size == part_size + assert upload_session.total_parts == total_parts + assert upload_session.num_parts_processed == num_parts_processed + assert upload_session.type == upload_session_type + assert upload_session.id == upload_session_id + assert upload_session._use_upload_session_urls == use_upload_session_urls + + @pytest.fixture() def mock_items_response_with_marker(mock_items): # pylint:disable=redefined-outer-name @@ -334,37 +375,6 @@ def test_upload_combines_preflight_and_accelerator_calls_if_both_are_requested( mock_box_session.options.assert_called_once() -def test_create_upload_session(test_folder, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions' - file_size = 197520 - file_name = 'test_file.pdf' - upload_session_id = 'F971964745A5CD0C001BBE4E58196BFD' - upload_session_type = 'upload_session' - num_parts_processed = 0 - total_parts = 16 - part_size = 12345 - expected_data = { - 'folder_id': test_folder.object_id, - 'file_size': file_size, - 'file_name': file_name, - } - mock_box_session.post.return_value.json.return_value = { - 'id': upload_session_id, - 'type': upload_session_type, - 'num_parts_processed': num_parts_processed, - 'total_parts': total_parts, - 'part_size': part_size, - } - upload_session = test_folder.create_upload_session(file_size, file_name) - mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data)) - assert isinstance(upload_session, UploadSession) - assert upload_session.part_size == part_size - assert upload_session.total_parts == total_parts - assert upload_session.num_parts_processed == num_parts_processed - assert upload_session.type == upload_session_type - assert upload_session.id == upload_session_id - - def test_upload_stream_does_preflight_check_if_specified( mock_box_session, test_folder, diff --git a/test/unit/object/test_upload_session.py b/test/unit/object/test_upload_session.py index 1b2e63c5..ff66e90c 100644 --- a/test/unit/object/test_upload_session.py +++ b/test/unit/object/test_upload_session.py @@ -4,6 +4,7 @@ import io import json import pytest +from pytest_lazyfixture import lazy_fixture from boxsdk.exception import BoxAPIException from boxsdk.config import API @@ -12,17 +13,55 @@ from boxsdk.object.upload_session import UploadSession +UPLOAD_SESSION_ID = 'F971964745A5CD0C001BBE4E58196BFD' +SESSION_ENDPOINTS = { + "abort": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}", + "commit": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}/commit", + "list_parts": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}/parts", + "log_event": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}/log", + "status": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}", + "upload_part": f"https://changed.box.com/api/2.0/files/upload_sessions/{UPLOAD_SESSION_ID}" +} + + @pytest.fixture() -def test_upload_session(mock_box_session): +def upload_session_using_upload_session_urls(mock_box_session): upload_session_response_object = { 'part_size': 8, 'total_parts': 10, + 'session_endpoints': SESSION_ENDPOINTS, } - return UploadSession(mock_box_session, '11493C07ED3EABB6E59874D3A1EF3581', upload_session_response_object) + return UploadSession( + mock_box_session, + UPLOAD_SESSION_ID, + upload_session_response_object, + use_upload_session_urls=True + ) -def test_get_parts(test_upload_session, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}/parts' +@pytest.fixture() +def upload_session_not_using_upload_session_urls(mock_box_session): + upload_session_response_object = { + 'part_size': 8, + 'total_parts': 10, + 'session_endpoints': SESSION_ENDPOINTS, + } + return UploadSession( + mock_box_session, + UPLOAD_SESSION_ID, + upload_session_response_object, + use_upload_session_urls=False + ) + + +@pytest.mark.parametrize( + 'test_upload_session, expected_url', + [ + (lazy_fixture('upload_session_using_upload_session_urls'), SESSION_ENDPOINTS['list_parts']), + (lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/parts') + ]) +def test_get_parts(mock_box_session, test_upload_session, expected_url): mock_entry = { 'part_id': '8F0966B1', 'offset': 0, @@ -44,16 +83,28 @@ def test_get_parts(test_upload_session, mock_box_session): assert part['offset'] == mock_entry['offset'] -def test_abort(test_upload_session, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}' +@pytest.mark.parametrize( + 'test_upload_session, expected_url', + [ + (lazy_fixture('upload_session_using_upload_session_urls'), SESSION_ENDPOINTS['abort']), + (lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}') + ]) +def test_abort(mock_box_session, test_upload_session, expected_url): mock_box_session.delete.return_value.ok = True result = test_upload_session.abort() - mock_box_session.delete.assert_called_once_with(expected_url, expect_json_response=False, headers=None, params={}) + mock_box_session.delete.assert_called_once_with(expected_url, expect_json_response=False) assert result is True -def test_upload_part_bytes(test_upload_session, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}' +@pytest.mark.parametrize( + 'test_upload_session, expected_url', + [ + (lazy_fixture('upload_session_using_upload_session_urls'), SESSION_ENDPOINTS['upload_part']), + (lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}') + ]) +def test_upload_part_bytes(mock_box_session, test_upload_session, expected_url): part_bytes = b'abcdefgh' offset = 32 total_size = 80 @@ -80,8 +131,14 @@ def test_upload_part_bytes(test_upload_session, mock_box_session): assert part['part_id'] == 'ABCDEF123' -def test_commit(test_upload_session, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}/commit' +@pytest.mark.parametrize( + 'test_upload_session, expected_url', + [ + (lazy_fixture('upload_session_using_upload_session_urls'), SESSION_ENDPOINTS['commit']), + (lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/commit') + ]) +def test_commit(mock_box_session, test_upload_session, expected_url): sha1 = hashlib.sha1() sha1.update(b'fake_file_data') file_id = '12345' @@ -120,17 +177,33 @@ def test_commit(test_upload_session, mock_box_session): }, ], } - created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag) - mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + created_file = test_upload_session.commit( + content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag + ) + mock_box_session.post.assert_called_once_with( + expected_url, data=json.dumps(expected_data), headers=expected_headers + ) assert isinstance(created_file, File) assert created_file.id == file_id assert created_file.type == file_type assert created_file.description == 'This is a test description.' -def test_commit_with_missing_params(test_upload_session, mock_box_session): - expected_get_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}/parts' - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}/commit' +@pytest.mark.parametrize( + 'test_upload_session, expected_get_url, expected_commit_url', + [ + ( + lazy_fixture('upload_session_using_upload_session_urls'), + SESSION_ENDPOINTS['list_parts'], + SESSION_ENDPOINTS['commit'] + ), + ( + lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/parts', + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/commit' + ) + ]) +def test_commit_with_missing_params(mock_box_session, test_upload_session, expected_get_url, expected_commit_url): sha1 = hashlib.sha1() sha1.update(b'fake_file_data') file_id = '12345' @@ -172,14 +245,22 @@ def test_commit_with_missing_params(test_upload_session, mock_box_session): } created_file = test_upload_session.commit(content_sha1=sha1.digest()) mock_box_session.get.assert_called_once_with(expected_get_url, params={'offset': None}) - mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + mock_box_session.post.assert_called_once_with( + expected_commit_url, data=json.dumps(expected_data), headers=expected_headers + ) assert isinstance(created_file, File) assert created_file.id == file_id assert created_file.type == file_type -def test_commit_returns_none_when_202_is_returned(test_upload_session, mock_box_session): - expected_url = f'{API.UPLOAD_URL}/files/upload_sessions/{test_upload_session.object_id}/commit' +@pytest.mark.parametrize( + 'test_upload_session, expected_url', + [ + (lazy_fixture('upload_session_using_upload_session_urls'), SESSION_ENDPOINTS['commit']), + (lazy_fixture('upload_session_not_using_upload_session_urls'), + f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/commit') + ]) +def test_commit_returns_none_when_202_is_returned(mock_box_session, test_upload_session, expected_url): sha1 = hashlib.sha1() sha1.update(b'fake_file_data') file_etag = '7' @@ -209,12 +290,22 @@ def test_commit_returns_none_when_202_is_returned(test_upload_session, mock_box_ } mock_box_session.post.side_effect = BoxAPIException(status=202) - created_file = test_upload_session.commit(content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag) + created_file = test_upload_session.commit( + content_sha1=sha1.digest(), parts=parts, file_attributes=file_attributes, etag=file_etag + ) - mock_box_session.post.assert_called_once_with(expected_url, data=json.dumps(expected_data), headers=expected_headers) + mock_box_session.post.assert_called_once_with( + expected_url, data=json.dumps(expected_data), headers=expected_headers + ) assert created_file is None +@pytest.mark.parametrize( + 'test_upload_session', + [ + lazy_fixture('upload_session_using_upload_session_urls'), + lazy_fixture('upload_session_not_using_upload_session_urls'), + ]) def test_get_chunked_uploader_for_stream(test_upload_session): file_size = 197520 part_bytes = b'abcdefgh' @@ -223,6 +314,12 @@ def test_get_chunked_uploader_for_stream(test_upload_session): assert isinstance(chunked_uploader, ChunkedUploader) +@pytest.mark.parametrize( + 'test_upload_session', + [ + lazy_fixture('upload_session_using_upload_session_urls'), + lazy_fixture('upload_session_not_using_upload_session_urls'), + ]) def test_get_chunked_uploader(mock_content_response, mock_file_path, test_upload_session): mock_file_stream = io.BytesIO(mock_content_response.content) file_size = 197520 @@ -231,3 +328,21 @@ def test_get_chunked_uploader(mock_content_response, mock_file_path, test_upload with patch('boxsdk.object.upload_session.open', return_value=mock_file_stream): chunked_uploader = test_upload_session.get_chunked_uploader(mock_file_path) assert isinstance(chunked_uploader, ChunkedUploader) + + +def test_get_url_do_not_use_session_urls_if_base_url_was_changed(upload_session_using_upload_session_urls): + old_base_upload_url = API.UPLOAD_URL + new_base_upload_url = 'https://new-upload.box.com/api/2.0' + API.UPLOAD_URL = new_base_upload_url + try: + url = upload_session_using_upload_session_urls.get_url('commit', url_key='commit') + assert url == f'{new_base_upload_url}/files/upload_sessions/{UPLOAD_SESSION_ID}/commit' + assert url != SESSION_ENDPOINTS['commit'] + finally: + API.UPLOAD_URL = old_base_upload_url + + +def test_get_url_uses_session_urls_if_base_url_was_not_changed(upload_session_using_upload_session_urls): + url = upload_session_using_upload_session_urls.get_url('commit', url_key='commit') + assert url != f'{API.UPLOAD_URL}/files/upload_sessions/{UPLOAD_SESSION_ID}/commit' + assert url == SESSION_ENDPOINTS['commit']