From 79a625b17c5127ac4198ee47b9aad0d20cf4cbf5 Mon Sep 17 00:00:00 2001 From: yacchin1205 Date: Wed, 3 May 2017 10:23:18 +0900 Subject: [PATCH 1/3] Add swift addon --- requirements.txt | 4 + setup.py | 1 + tests/providers/swift/__init__.py | 0 tests/providers/swift/test_provider.py | 442 ++++++++++++++++++++++++ waterbutler/providers/swift/LICENSE | 203 +++++++++++ waterbutler/providers/swift/README.md | 9 + waterbutler/providers/swift/__init__.py | 1 + waterbutler/providers/swift/metadata.py | 100 ++++++ waterbutler/providers/swift/provider.py | 300 ++++++++++++++++ waterbutler/providers/swift/settings.py | 3 + 10 files changed, 1063 insertions(+) create mode 100644 tests/providers/swift/__init__.py create mode 100644 tests/providers/swift/test_provider.py create mode 100644 waterbutler/providers/swift/LICENSE create mode 100644 waterbutler/providers/swift/README.md create mode 100644 waterbutler/providers/swift/__init__.py create mode 100644 waterbutler/providers/swift/metadata.py create mode 100644 waterbutler/providers/swift/provider.py create mode 100644 waterbutler/providers/swift/settings.py diff --git a/requirements.txt b/requirements.txt index 91b29b939..6847e3597 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,7 @@ certifi==2015.4.28 # Analytics requirements python-geoip-geolite2==2015.0303 + +# Swift +python-keystoneclient==3.10.0 +python-swiftclient==3.3.0 diff --git a/setup.py b/setup.py index a530f8163..bd02fbdc5 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def parse_requirements(requirements): 'dataverse = waterbutler.providers.dataverse:DataverseProvider', 'box = waterbutler.providers.box:BoxProvider', 'googledrive = waterbutler.providers.googledrive:GoogleDriveProvider', + 'swift = waterbutler.providers.swift:SwiftProvider', ], 'waterbutler.providers.tasks': [ 'osfstorage_parity = waterbutler.providers.osfstorage.tasks.parity', diff --git a/tests/providers/swift/__init__.py b/tests/providers/swift/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/providers/swift/test_provider.py b/tests/providers/swift/test_provider.py new file mode 100644 index 000000000..2887d36fb --- /dev/null +++ b/tests/providers/swift/test_provider.py @@ -0,0 +1,442 @@ +import pytest + +import io +import time +import base64 +import hashlib +from http import client +from unittest import mock + +import aiohttpretty + +from waterbutler.core import streams +from waterbutler.core import metadata +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath + +from waterbutler.providers.swift import SwiftProvider +from waterbutler.providers.swift.metadata import SwiftFileMetadata +from waterbutler.providers.swift.metadata import SwiftFolderMetadata +from swiftclient import quote + + +@pytest.fixture +def auth(): + return { + 'name': 'cat', + 'email': 'cat@cat.com', + } + + +@pytest.fixture +def credentials(): + return { + 'username': 'Dont dead', + 'password': 'open inside', + 'tenant_name': 'test', + 'auth_url': 'http://test_url/v2.0' + } + + +@pytest.fixture +def settings(): + return { + 'container': 'that kerning' + } + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock(return_value=1454684930.0) + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def provider(auth, credentials, settings): + provider = SwiftProvider(auth, credentials, settings) + return provider + + +@pytest.fixture +def file_content(): + return b'sleepy' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + +@pytest.fixture +def folder_metadata(): + return b'''[ + { + "hash": "3def40db06680692d01f44f2fd12066c", + "last_modified": "2017-02-09T08:10:51.828100", + "bytes": 70227, + "name": "mendeley_cistyle_osf_nii_ac_jp_2.png", + "content_type": "image/png" + }, + { + "hash": "d9a3fdfc7ca17c47ed007bed5d2eb873", + "last_modified": "2017-02-07T23:09:24.057080", + "bytes": 9, + "name": "Photos/test.txt", + "content_type": "text/plain" + }, + { + "hash": "d9a3fdfc7ca17c47ed007bed5d2eb873", + "last_modified": "2017-02-07T23:09:24.057080", + "bytes": 9, + "name": "Photos/a/test.txt", + "content_type": "text/plain" + }, + { + "hash": "d9a3fdfc7ca17c47ed007bed5d2eb873", + "last_modified": "2017-02-07T23:09:24.057080", + "bytes": 9, + "name": "Photos/.osfkeep", + "content_type": "text/plain" + }, + { + "hash": "d9a3fdfc7ca17c47ed007bed5d2eb873", + "last_modified": "2017-02-07T14:09:55.351480", + "bytes": 9, + "name": "test.txt", + "content_type": "text/plain" + } +]''' + + +@pytest.fixture +def file_metadata(): + return { + 'Content-Length': '9', + 'Content-Type': 'text/plain', + 'Last-Modified': 'Tue, 07 Feb 2017 14:09:56 GMT', + 'Etag': 'd9a3fdfc7ca17c47ed007bed5d2eb873', + 'X-Timestamp': '1486476595.35148', + 'X-Object-Meta-Mtime': '1486467323.087638', + 'X-Trans-Id': 'txb0f6ed12846f422d9afd0-0058a5ac17' + } + + +class TestValidatePath: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_validate_v1_path_file(self, provider, file_metadata, + mock_time): + file_path = 'foobah' + provider.url = 'http://test_url' + provider.token = 'test' + good_metadata_url = provider.generate_url(file_path) + bad_metadata_url = provider.generate_url() + aiohttpretty.register_uri('HEAD', good_metadata_url, headers=file_metadata) + aiohttpretty.register_uri('GET', bad_metadata_url, params={'format': 'json'}, status=404) + + try: + wb_path_v1 = await provider.validate_v1_path('/' + file_path) + except Exception as exc: + pytest.fail(str(exc)) + + with pytest.raises(exceptions.NotFoundError) as exc: + await provider.validate_v1_path('/' + file_path + '/') + + assert exc.value.code == client.NOT_FOUND + + wb_path_v0 = await provider.validate_path('/' + file_path) + + assert wb_path_v1 == wb_path_v0 + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_validate_v1_path_folder(self, provider, folder_metadata, mock_time): + folder_path = 'Photos' + + provider.url = 'http://test_url' + provider.token = 'test' + good_metadata_url = provider.generate_url() + bad_metadata_url = provider.generate_url(folder_path) + aiohttpretty.register_uri( + 'GET', good_metadata_url, params={'format': 'json'}, + body=folder_metadata, headers={'Content-Type': 'application/json'} + ) + aiohttpretty.register_uri('HEAD', bad_metadata_url, status=404) + + try: + wb_path_v1 = await provider.validate_v1_path('/' + folder_path + '/') + except Exception as exc: + pytest.fail(str(exc)) + + with pytest.raises(exceptions.NotFoundError) as exc: + await provider.validate_v1_path('/' + folder_path) + + assert exc.value.code == client.NOT_FOUND + + wb_path_v0 = await provider.validate_path('/' + folder_path + '/') + + assert wb_path_v1 == wb_path_v0 + + @pytest.mark.asyncio + async def test_normal_name(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/path.txt') + assert path.name == 'path.txt' + assert path.parent.name == 'a' + assert path.is_file + assert not path.is_dir + assert not path.is_root + + @pytest.mark.asyncio + async def test_folder(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/folder/') + assert path.name == 'folder' + assert path.parent.name == 'a' + assert not path.is_file + assert path.is_dir + assert not path.is_root + + @pytest.mark.asyncio + async def test_root(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/folder/') + assert path.name == 'folder' + assert path.parent.name == 'a' + assert not path.is_file + assert path.is_dir + assert not path.is_root + + +class TestCRUD: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download(self, provider, mock_time): + path = WaterButlerPath('/muhtriangle') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url(path.path) + aiohttpretty.register_uri('GET', url, body=b'delicious', auto_length=True) + + result = await provider.download(path) + content = await result.read() + + assert content == b'delicious' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download_folder_400s(self, provider, mock_time): + with pytest.raises(exceptions.DownloadError) as e: + await provider.download(WaterButlerPath('/cool/folder/mom/')) + assert e.value.code == 400 + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete(self, provider, mock_time): + path = WaterButlerPath('/some-file') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url(path.path) + aiohttpretty.register_uri('DELETE', url, status=200) + + await provider.delete(path) + + assert aiohttpretty.has_call(method='DELETE', uri=url) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_folder_delete(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/Photos/') + + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url() + aiohttpretty.register_uri( + 'GET', url, params={'format': 'json'}, + body=folder_metadata, headers={'Content-Type': 'application/json'} + ) + delete_urls = [provider.generate_url(path.path + "test.txt"), + provider.generate_url(path.path + ".osfkeep"), + provider.generate_url(path.path + "a/test.txt")] + for delete_url in delete_urls: + aiohttpretty.register_uri('DELETE', delete_url, status=200) + + await provider.delete(path) + + assert aiohttpretty.has_call(method='DELETE', uri=delete_urls[0]) + assert aiohttpretty.has_call(method='DELETE', uri=delete_urls[1]) + assert aiohttpretty.has_call(method='DELETE', uri=delete_urls[2]) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_upload(self, provider, file_content, file_stream, file_metadata, mock_time): + path = WaterButlerPath('/foobah') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url(path.path) + aiohttpretty.register_uri('PUT', url, status=200) + metadata_url = provider.generate_url(path.path) + aiohttpretty.register_uri( + 'HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_metadata}, + ], + ) + + metadata, created = await provider.upload(file_stream, path) + + assert metadata.kind == 'file' + assert created + assert aiohttpretty.has_call(method='PUT', uri=url) + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + + +class TestMetadata: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_root(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/') + assert path.is_root + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url() + aiohttpretty.register_uri('GET', url, params={'format': 'json'}, body=folder_metadata, + headers={'Content-Type': 'application/json'}) + + result = await provider.metadata(path) + + assert isinstance(result, list) + assert len(result) == 3 + assert result[0].path == '/Photos/' + assert result[0].name == 'Photos' + assert result[0].is_folder + + assert result[1].path == '/mendeley_cistyle_osf_nii_ac_jp_2.png' + assert result[1].name == 'mendeley_cistyle_osf_nii_ac_jp_2.png' + assert not result[1].is_folder + assert result[1].extra['md5'] == '3def40db06680692d01f44f2fd12066c' + + assert result[2].path == '/test.txt' + assert result[2].name == 'test.txt' + assert not result[2].is_folder + assert result[2].extra['md5'] == 'd9a3fdfc7ca17c47ed007bed5d2eb873' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_folder(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/Photos/') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url() + aiohttpretty.register_uri('GET', url, params={'format': 'json'}, body=folder_metadata, + headers={'Content-Type': 'application/json'}) + + result = await provider.metadata(path) + + assert isinstance(result, list) + assert len(result) == 2 + assert result[0].path == '/Photos/a/' + assert result[0].name == 'a' + assert result[0].is_folder + + assert result[1].path == '/Photos/test.txt' + assert result[1].name == 'test.txt' + assert not result[1].is_folder + assert result[1].extra['md5'] == 'd9a3fdfc7ca17c47ed007bed5d2eb873' + + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_file(self, provider, file_metadata, mock_time): + path = WaterButlerPath('/Foo/Bar/my-image.jpg') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url(path.path) + aiohttpretty.register_uri('HEAD', url, headers=file_metadata) + + result = await provider.metadata(path) + + assert isinstance(result, metadata.BaseFileMetadata) + assert result.path == str(path) + assert result.name == 'my-image.jpg' + assert result.extra['md5'] == 'd9a3fdfc7ca17c47ed007bed5d2eb873' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_file_missing(self, provider, mock_time): + path = WaterButlerPath('/notfound.txt') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url(path.path) + aiohttpretty.register_uri('HEAD', url, status=404) + + with pytest.raises(exceptions.MetadataError): + await provider.metadata(path) + + +class TestCreateFolder: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_must_start_with_slash(self, provider, mock_time): + path = WaterButlerPath('/alreadyexists') + + with pytest.raises(exceptions.CreateFolderError) as e: + await provider.create_folder(path) + + assert e.value.code == 400 + assert e.value.message == 'Path must be a directory' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_errors_conflict(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/alreadyexists/') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url() + aiohttpretty.register_uri('GET', url, + params={'format': 'json'}, + body=folder_metadata, + headers={'Content-Type': 'application/json'}) + url = provider.generate_url('alreadyexists') + aiohttpretty.register_uri('HEAD', url, status=200) + url = provider.generate_url('alreadyexists/.osfkeep') + aiohttpretty.register_uri('PUT', url, status=200) + + with pytest.raises(exceptions.FolderNamingConflict) as e: + await provider.create_folder(path) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_creates(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/doesntalreadyexists/') + provider.url = 'http://test_url' + provider.token = 'test' + url = provider.generate_url() + aiohttpretty.register_uri('GET', url, + params={'format': 'json'}, + body=folder_metadata, + headers={'Content-Type': 'application/json'}) + url = provider.generate_url('doesntalreadyexists') + aiohttpretty.register_uri('HEAD', url, status=404) + url = provider.generate_url('doesntalreadyexists/.osfkeep') + aiohttpretty.register_uri('PUT', url, status=200) + + resp = await provider.create_folder(path) + + assert resp.kind == 'folder' + assert resp.name == 'doesntalreadyexists' + assert resp.path == '/doesntalreadyexists/' + + +class TestOperations: + + async def test_equality(self, provider, mock_time): + assert provider.can_intra_copy(provider) + assert provider.can_intra_move(provider) diff --git a/waterbutler/providers/swift/LICENSE b/waterbutler/providers/swift/LICENSE new file mode 100644 index 000000000..55c6b3044 --- /dev/null +++ b/waterbutler/providers/swift/LICENSE @@ -0,0 +1,203 @@ + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 National Institute of Informatics + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/waterbutler/providers/swift/README.md b/waterbutler/providers/swift/README.md new file mode 100644 index 000000000..2ad5149bb --- /dev/null +++ b/waterbutler/providers/swift/README.md @@ -0,0 +1,9 @@ +# WaterButler Swift provider: Custom WaterButler providers for OSF in Japan + +## License + +[Apache License Version 2.0](LICENSE) © 2017 National Institute of Informatics + +## Setting up Swift provider + +No settings for the provider. diff --git a/waterbutler/providers/swift/__init__.py b/waterbutler/providers/swift/__init__.py new file mode 100644 index 000000000..648fc2423 --- /dev/null +++ b/waterbutler/providers/swift/__init__.py @@ -0,0 +1 @@ +from .provider import SwiftProvider # noqa diff --git a/waterbutler/providers/swift/metadata.py b/waterbutler/providers/swift/metadata.py new file mode 100644 index 000000000..402eafa46 --- /dev/null +++ b/waterbutler/providers/swift/metadata.py @@ -0,0 +1,100 @@ +import os + +from waterbutler.core import metadata +from swiftclient import parse_header_string + + +def resp_headers(headers): + return dict(map(lambda item: (parse_header_string(item[0]).lower(), + parse_header_string(item[1])), + headers.items())) + + +class SwiftMetadata(metadata.BaseMetadata): + + @property + def provider(self): + return 'swift' + + @property + def name(self): + return os.path.split(self.path)[1] + + @property + def created_utc(self): + return None + + +class SwiftFileMetadataHeaders(SwiftMetadata, metadata.BaseFileMetadata): + + def __init__(self, path, headers): + self._path = path + # Cast to dict to clone as the headers will + # be destroyed when the request leaves scope + super().__init__(resp_headers(headers)) + + @property + def path(self): + return '/' + self._path + + @property + def size(self): + return self.raw['content-length'] + + @property + def content_type(self): + return self.raw['content-type'] + + @property + def modified(self): + return self.raw['last-modified'] + + @property + def etag(self): + return self.raw['etag'] + + @property + def extra(self): + return { + 'md5': self.raw['etag'] + } + + +class SwiftFileMetadata(SwiftMetadata, metadata.BaseFileMetadata): + + @property + def path(self): + return '/' + self.raw['name'] + + @property + def size(self): + return int(self.raw['bytes']) + + @property + def modified(self): + return self.raw['last_modified'] + + @property + def content_type(self): + return self.raw['content_type'] + + @property + def etag(self): + return self.raw['hash'] + + @property + def extra(self): + return { + 'md5': self.raw['hash'] + } + + +class SwiftFolderMetadata(SwiftMetadata, metadata.BaseFolderMetadata): + + @property + def name(self): + return self.raw['prefix'].split('/')[-2] + + @property + def path(self): + return '/' + self.raw['prefix'] diff --git a/waterbutler/providers/swift/provider.py b/waterbutler/providers/swift/provider.py new file mode 100644 index 000000000..6fd095ffe --- /dev/null +++ b/waterbutler/providers/swift/provider.py @@ -0,0 +1,300 @@ +import hashlib +import functools + +from swiftclient import Connection, quote +from swiftclient.utils import parse_api_response + +from waterbutler.core import streams +from waterbutler.core import provider +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath + +from waterbutler.providers.swift.metadata import SwiftFileMetadata +from waterbutler.providers.swift.metadata import SwiftFolderMetadata +from waterbutler.providers.swift.metadata import SwiftFileMetadataHeaders +from waterbutler.providers.swift.metadata import resp_headers + + +class SwiftProvider(provider.BaseProvider): + """Provider for Swift cloud storage service. + """ + NAME = 'swift' + + def __init__(self, auth, credentials, settings): + """ + :param dict auth: Not used + :param dict credentials: Dict containing `username`, `password` and `tenant_name` + :param dict settings: Dict containing `container` + """ + super().__init__(auth, credentials, settings) + + self.connection = Connection(auth_version='2', + authurl=credentials['auth_url'], + user=credentials['username'], + key=credentials['password'], + tenant_name=credentials['tenant_name']) + self.url = None + self.token = None + + self.container = settings['container'] + + @property + def default_headers(self): + if not self.url or not self.token: + self.url, self.token = self.connection.get_auth() + return {'X-Auth-Token': self.token} + + def generate_url(self, name=None): + if not self.url or not self.token: + self.url, self.token = self.connection.get_auth() + if name is None: + return '%s/%s' % (self.url, quote(self.container)) + else: + return '%s/%s/%s' % (self.url, quote(self.container), quote(name)) + + async def validate_v1_path(self, path, **kwargs): + if path == '/': + return WaterButlerPath(path) + + implicit_folder = path.endswith('/') + + assert path.startswith('/') + if implicit_folder: + resp = await self.make_request( + 'GET', + self.generate_url, + params={'format': 'json'}, + expects=(200, 404), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + if resp.status == 404: + raise exceptions.NotFoundError(str(path)) + objects = parse_api_response(resp_headers(resp.headers), respbody) + if len(list(filter(lambda o: o['name'].startswith(path[1:]), + objects))) == 0: + raise exceptions.NotFoundError(str(path)) + else: + resp = await self.make_request( + 'HEAD', + functools.partial(self.generate_url, path[1:]), + expects=(200, 404), + throws=exceptions.MetadataError, + ) + await resp.release() + if resp.status == 404: + raise exceptions.NotFoundError(str(path)) + + return WaterButlerPath(path) + + async def validate_path(self, path, **kwargs): + return WaterButlerPath(path) + + def can_duplicate_names(self): + return True + + def can_intra_copy(self, dest_provider, path=None): + # Not supported + return False + + def can_intra_move(self, dest_provider, path=None): + # Not supported + return False + + async def intra_copy(self, dest_provider, source_path, dest_path): + # Not supported + raise NotImplementedError() + + async def download(self, path, accept_url=False, version=None, range=None, **kwargs): + """ + :param str path: Path to the key you want to download + :param dict \*\*kwargs: Additional arguments that are ignored + :rtype: :class:`waterbutler.core.streams.ResponseStreamReader` + :raises: :class:`waterbutler.core.exceptions.DownloadError` + """ + + if not path.is_file: + raise exceptions.DownloadError('No file specified for download', code=400) + + assert not path.path.startswith('/') + url = functools.partial(self.generate_url, path.path) + + resp = await self.make_request( + 'GET', + url, + expects=(200, ), + throws=exceptions.MetadataError, + ) + + return streams.ResponseStreamReader(resp) + + async def upload(self, stream, path, conflict='replace', **kwargs): + """Uploads the given stream to Swift + + :param waterbutler.core.streams.RequestWrapper stream: The stream to put to Swift + :param str path: The full path of the key to upload to/into + + :rtype: dict, bool + """ + + path, exists = await self.handle_name_conflict(path, conflict=conflict) + stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5)) + headers = {'Content-Length': str(stream.size)} + + assert not path.path.startswith('/') + + resp = await self.make_request( + 'PUT', + functools.partial(self.generate_url, path.path), + data=stream, + headers=headers, + skip_auto_headers={'CONTENT-TYPE'}, + expects=(200, 201, 202, ), + throws=exceptions.UploadError, + ) + await resp.release() + + return (await self.metadata(path, **kwargs)), not exists + + async def delete(self, path, confirm_delete=0, **kwargs): + """Deletes the key at the specified path + + :param str path: The path of the key to delete + :param int confirm_delete: Must be 1 to confirm root folder delete + """ + + if path.is_root: + if not confirm_delete == 1: + raise exceptions.DeleteError( + 'confirm_delete=1 is required for deleting root provider folder', + code=400 + ) + + if path.is_file: + assert not path.path.startswith('/') + resp = await self.make_request( + 'DELETE', + functools.partial(self.generate_url, path.path), + expects=(200, 202, 204, 404), + throws=exceptions.MetadataError, + ) + await resp.release() + else: + await self._delete_folder(path, **kwargs) + + async def _delete_folder(self, path, **kwargs): + resp = await self.make_request( + 'GET', + self.generate_url, + params={'format': 'json'}, + expects=(200, ), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + objects = list(map(lambda o: (o['name'][len(path.path):], o), + filter(lambda o: o['name'].startswith(path.path), + parse_api_response(resp_headers(resp.headers), + respbody)))) + if len(objects) == 0 and not path.is_root: + raise exceptions.DeleteError('Not found', code=404) + for name, obj in objects: + resp = await self.make_request( + 'DELETE', + functools.partial(self.generate_url, obj['name']), + expects=(200, 202, 204, 404), + throws=exceptions.MetadataError, + ) + await resp.release() + + async def revisions(self, path, **kwargs): + """Get past versions of the requested key + + :param str path: The path to a key + :rtype list: + """ + return [] + + async def metadata(self, path, revision=None, **kwargs): + """Get Metadata about the requested file or folder + + :param WaterButlerPath path: The path to a key or folder + :rtype: dict or list + """ + if path.is_dir: + return (await self._metadata_folder(path)) + + return (await self._metadata_file(path, revision=revision)) + + async def create_folder(self, path, folder_precheck=True, **kwargs): + """ + :param str path: The path to create a folder at + """ + + WaterButlerPath.validate_folder(path) + + if folder_precheck: + if (await self.exists(path)): + raise exceptions.FolderNamingConflict(str(path)) + if (await self.exists(await self.validate_path('/' + path.path[:-1]))): + raise exceptions.FolderNamingConflict(str(path)) + + resp = await self.make_request( + 'PUT', + functools.partial(self.generate_url, path.path + '.osfkeep'), + data='', + skip_auto_headers={'CONTENT-TYPE'}, + expects=(200, 201, 202, ), + throws=exceptions.CreateFolderError + ) + await resp.release() + + return SwiftFolderMetadata({'prefix': path.path}) + + async def _metadata_file(self, path, revision=None): + if revision == 'Latest': + revision = None + assert not path.path.startswith('/') + resp = await self.make_request( + 'HEAD', + functools.partial(self.generate_url, path.path), + expects=(200, ), + throws=exceptions.MetadataError, + ) + await resp.release() + return SwiftFileMetadataHeaders(path.path, resp.headers) + + async def _metadata_folder(self, path): + resp = await self.make_request( + 'GET', + self.generate_url, + params={'format': 'json'}, + expects=(200, ), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + objects = list(map(lambda o: (o['name'][len(path.path):], o), + filter(lambda o: o['name'].startswith(path.path), + parse_api_response(resp_headers(resp.headers), + respbody)))) + if len(objects) == 0 and not path.is_root: + raise exceptions.MetadataError('Not found', code=404) + + contents = list(filter(lambda o: '/' not in o[0], objects)) + prefixes = sorted(set(map(lambda o: path.path + o[0][:o[0].index('/') + 1], + filter(lambda o: '/' in o[0], objects)))) + + items = [ + SwiftFolderMetadata({'prefix': item}) + for item in prefixes + ] + + for content_path, content in contents: + if content_path == path.path: + continue + + fmetadata = SwiftFileMetadata(content) + if fmetadata.name == '.osfkeep': + continue + items.append(fmetadata) + + return items diff --git a/waterbutler/providers/swift/settings.py b/waterbutler/providers/swift/settings.py new file mode 100644 index 000000000..0748e1733 --- /dev/null +++ b/waterbutler/providers/swift/settings.py @@ -0,0 +1,3 @@ +from waterbutler import settings + +config = settings.child('NIISWIFT_PROVIDER_CONFIG') From 7e2c176c5fa3cf907952b62d7c8a6865b6cddcd3 Mon Sep 17 00:00:00 2001 From: yacchin1205 Date: Fri, 5 May 2017 07:54:54 +0900 Subject: [PATCH 2/3] Fix settings name --- waterbutler/providers/swift/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/swift/settings.py b/waterbutler/providers/swift/settings.py index 0748e1733..ed4031b3f 100644 --- a/waterbutler/providers/swift/settings.py +++ b/waterbutler/providers/swift/settings.py @@ -1,3 +1,3 @@ from waterbutler import settings -config = settings.child('NIISWIFT_PROVIDER_CONFIG') +config = settings.child('SWIFT_PROVIDER_CONFIG') From 6e390c488b3c089e9a5b662eaeb12880e8b6fe22 Mon Sep 17 00:00:00 2001 From: yacchin1205 Date: Sat, 6 May 2017 07:11:08 +0900 Subject: [PATCH 3/3] Add Azure Blob Storage provider --- requirements.txt | 3 + setup.py | 1 + tests/providers/azureblobstorage/__init__.py | 0 .../azureblobstorage/test_provider.py | 445 ++++++++++++++++++ .../providers/azureblobstorage/LICENSE | 203 ++++++++ .../providers/azureblobstorage/README.md | 9 + .../providers/azureblobstorage/__init__.py | 1 + .../providers/azureblobstorage/metadata.py | 93 ++++ .../providers/azureblobstorage/provider.py | 378 +++++++++++++++ .../providers/azureblobstorage/settings.py | 3 + 10 files changed, 1136 insertions(+) create mode 100644 tests/providers/azureblobstorage/__init__.py create mode 100644 tests/providers/azureblobstorage/test_provider.py create mode 100644 waterbutler/providers/azureblobstorage/LICENSE create mode 100644 waterbutler/providers/azureblobstorage/README.md create mode 100644 waterbutler/providers/azureblobstorage/__init__.py create mode 100644 waterbutler/providers/azureblobstorage/metadata.py create mode 100644 waterbutler/providers/azureblobstorage/provider.py create mode 100644 waterbutler/providers/azureblobstorage/settings.py diff --git a/requirements.txt b/requirements.txt index 6847e3597..4f9aa927c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,6 @@ python-geoip-geolite2==2015.0303 # Swift python-keystoneclient==3.10.0 python-swiftclient==3.3.0 + +# Azure Blob Storage +azure-storage==0.33.0 diff --git a/setup.py b/setup.py index bd02fbdc5..4af2617f5 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ def parse_requirements(requirements): 'box = waterbutler.providers.box:BoxProvider', 'googledrive = waterbutler.providers.googledrive:GoogleDriveProvider', 'swift = waterbutler.providers.swift:SwiftProvider', + 'azureblobstorage = waterbutler.providers.azureblobstorage:AzureBlobStorageProvider', ], 'waterbutler.providers.tasks': [ 'osfstorage_parity = waterbutler.providers.osfstorage.tasks.parity', diff --git a/tests/providers/azureblobstorage/__init__.py b/tests/providers/azureblobstorage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/providers/azureblobstorage/test_provider.py b/tests/providers/azureblobstorage/test_provider.py new file mode 100644 index 000000000..72373d679 --- /dev/null +++ b/tests/providers/azureblobstorage/test_provider.py @@ -0,0 +1,445 @@ +import pytest + +import io +import time +import base64 +import hashlib +from http import client +from unittest import mock + +import aiohttpretty + +from waterbutler.core import streams +from waterbutler.core import metadata +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath + +from waterbutler.providers.azureblobstorage import AzureBlobStorageProvider +from waterbutler.providers.azureblobstorage.metadata import AzureBlobStorageFileMetadata +from waterbutler.providers.azureblobstorage.metadata import AzureBlobStorageFolderMetadata + + +@pytest.fixture +def auth(): + return { + 'name': 'cat', + 'email': 'cat@cat.com', + } + + +@pytest.fixture +def credentials(): + return { + 'account_name': 'dontdead', + 'account_key': base64.b64encode(b'open inside'), + } + + +@pytest.fixture +def settings(): + return { + 'container': 'thatkerning' + } + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock(return_value=1454684930.0) + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def provider(auth, credentials, settings): + provider = AzureBlobStorageProvider(auth, credentials, settings) + return provider + + +@pytest.fixture +def file_content(): + return b'sleepy' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + +@pytest.fixture +def folder_metadata(): + return b''' + + + + Photos/test-text.txt + + Thu, 10 Nov 2016 11:04:45 GMT + 0x8D40959613D32F6 + 0 + text/plain + + + + + + BlockBlob + unlocked + available + + + + Photos/a/test.txt + + Thu, 10 Nov 2016 11:04:45 GMT + 0x8D40959613D32F6 + 0 + text/plain + + + + + + BlockBlob + unlocked + available + + + + top.txt + + Thu, 10 Nov 2016 11:04:45 GMT + 0x8D40959613D32F6 + 0 + text/plain + + + + + + BlockBlob + unlocked + available + + + + +''' + + +@pytest.fixture +def file_metadata(): + return { + 'CONTENT-LENGTH': '0', + 'CONTENT-TYPE': 'text/plain', + 'LAST-MODIFIED': 'Thu, 10 Nov 2016 11:04:45 GMT', + 'ACCEPT-RANGES': 'bytes', + 'ETAG': '"0x8D40959613D32F6"', + 'SERVER': 'Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0', + 'X-MS-REQUEST-ID': '5b4a3cb6-0001-00ea-4575-895e2c000000', + 'X-MS-VERSION': '2015-07-08', + 'X-MS-LEASE-STATUS': 'unlocked', + 'X-MS-LEASE-STATE': 'available', + 'X-MS-BLOB-TYPE': 'BlockBlob', + 'DATE': 'Fri, 17 Feb 2017 23:28:33 GMT' + } + + +class TestValidatePath: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_validate_v1_path_file(self, provider, file_metadata, + mock_time): + file_path = 'foobah' + for good_metadata_url in provider.generate_urls(file_path, secondary=True): + aiohttpretty.register_uri('HEAD', good_metadata_url, headers=file_metadata) + for bad_metadata_url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri('GET', bad_metadata_url, + params={'restype': 'container', 'comp': 'list'}, status=404) + + try: + wb_path_v1 = await provider.validate_v1_path('/' + file_path) + except Exception as exc: + pytest.fail(str(exc)) + + with pytest.raises(exceptions.NotFoundError) as exc: + await provider.validate_v1_path('/' + file_path + '/') + + assert exc.value.code == client.NOT_FOUND + + wb_path_v0 = await provider.validate_path('/' + file_path) + + assert wb_path_v1 == wb_path_v0 + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_validate_v1_path_folder(self, provider, folder_metadata, mock_time): + folder_path = 'Photos' + + for good_metadata_url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri( + 'GET', good_metadata_url, params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, headers={'Content-Type': 'application/xml'} + ) + for bad_metadata_url in provider.generate_urls(folder_path, secondary=True): + aiohttpretty.register_uri('HEAD', bad_metadata_url, status=404) + + try: + wb_path_v1 = await provider.validate_v1_path('/' + folder_path + '/') + except Exception as exc: + pytest.fail(str(exc)) + + with pytest.raises(exceptions.NotFoundError) as exc: + await provider.validate_v1_path('/' + folder_path) + + assert exc.value.code == client.NOT_FOUND + + wb_path_v0 = await provider.validate_path('/' + folder_path + '/') + + assert wb_path_v1 == wb_path_v0 + + @pytest.mark.asyncio + async def test_normal_name(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/path.txt') + assert path.name == 'path.txt' + assert path.parent.name == 'a' + assert path.is_file + assert not path.is_dir + assert not path.is_root + + @pytest.mark.asyncio + async def test_folder(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/folder/') + assert path.name == 'folder' + assert path.parent.name == 'a' + assert not path.is_file + assert path.is_dir + assert not path.is_root + + @pytest.mark.asyncio + async def test_root(self, provider, mock_time): + path = await provider.validate_path('/this/is/a/folder/') + assert path.name == 'folder' + assert path.parent.name == 'a' + assert not path.is_file + assert path.is_dir + assert not path.is_root + + +class TestCRUD: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download(self, provider, mock_time): + path = WaterButlerPath('/muhtriangle') + for url in provider.generate_urls(path.path, secondary=True): + aiohttpretty.register_uri('GET', url, body=b'delicious', auto_length=True) + + result = await provider.download(path) + content = await result.read() + + assert content == b'delicious' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download_folder_400s(self, provider, mock_time): + with pytest.raises(exceptions.DownloadError) as e: + await provider.download(WaterButlerPath('/cool/folder/mom/')) + assert e.value.code == 400 + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete(self, provider, mock_time): + path = WaterButlerPath('/some-file') + for url in provider.generate_urls(path.path): + aiohttpretty.register_uri('DELETE', url, status=200) + + await provider.delete(path) + + assert aiohttpretty.has_call(method='DELETE', uri=url) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_folder_delete(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/Photos/') + + for url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri( + 'GET', url, params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, headers={'Content-Type': 'application/xml'} + ) + delete_urls = [] + for url in provider.generate_urls(path.path + "test-text.txt"): + aiohttpretty.register_uri('DELETE', url, status=200) + delete_urls.append(url) + for url in provider.generate_urls(path.path + "a/test.txt"): + aiohttpretty.register_uri('DELETE', url, status=200) + delete_urls.append(url) + + await provider.delete(path) + + assert aiohttpretty.has_call(method='DELETE', uri=delete_urls[0]) + assert aiohttpretty.has_call(method='DELETE', uri=delete_urls[1]) + + +class TestMetadata: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_root(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/') + assert path.is_root + for url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri('GET', url, + params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, + headers={'Content-Type': 'application/xml'}) + + result = await provider.metadata(path) + + assert isinstance(result, list) + assert len(result) == 2 + assert result[0].path == '/Photos/' + assert result[0].name == 'Photos' + assert result[0].is_folder + + assert result[1].path == '/top.txt' + assert result[1].name == 'top.txt' + assert not result[1].is_folder + assert result[1].extra['md5'] == None + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_folder(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/Photos/') + for url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri('GET', url, + params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, + headers={'Content-Type': 'application/xml'}) + + result = await provider.metadata(path) + + assert isinstance(result, list) + assert len(result) == 2 + assert result[0].path == '/Photos/a/' + assert result[0].name == 'a' + assert result[0].is_folder + + assert result[1].path == '/Photos/test-text.txt' + assert result[1].name == 'test-text.txt' + assert not result[1].is_folder + assert result[1].extra['md5'] == None + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_file(self, provider, file_metadata, mock_time): + path = WaterButlerPath('/Foo/Bar/my-image.jpg') + provider.url = 'http://test_url' + provider.token = 'test' + for url in provider.generate_urls(path.path, secondary=True): + aiohttpretty.register_uri('HEAD', url, headers=file_metadata) + + result = await provider.metadata(path) + + assert isinstance(result, metadata.BaseFileMetadata) + assert result.path == str(path) + assert result.name == 'my-image.jpg' + assert result.modified is not None + assert result.extra['md5'] == None + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_file_missing(self, provider, mock_time): + path = WaterButlerPath('/notfound.txt') + provider.url = 'http://test_url' + provider.token = 'test' + for url in provider.generate_urls(path.path, secondary=True): + aiohttpretty.register_uri('HEAD', url, status=404) + + with pytest.raises(exceptions.MetadataError): + await provider.metadata(path) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_upload(self, provider, file_content, file_stream, file_metadata, mock_time): + path = WaterButlerPath('/foobah') + for url in provider.generate_urls(path.path): + aiohttpretty.register_uri('PUT', url, status=200) + for metadata_url in provider.generate_urls(path.path): + aiohttpretty.register_uri( + 'HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_metadata}, + ], + ) + + metadata, created = await provider.upload(file_stream, path) + + assert metadata.kind == 'file' + assert created + assert aiohttpretty.has_call(method='PUT', uri=url) + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + + +class TestCreateFolder: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_must_start_with_slash(self, provider, mock_time): + path = WaterButlerPath('/alreadyexists') + + with pytest.raises(exceptions.CreateFolderError) as e: + await provider.create_folder(path) + + assert e.value.code == 400 + assert e.value.message == 'Path must be a directory' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_errors_conflict(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/alreadyexists/') + for url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri('GET', url, + params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, + headers={'Content-Type': 'application/xml'}) + for url in provider.generate_urls('alreadyexists', secondary=True): + aiohttpretty.register_uri('HEAD', url, status=200) + for url in provider.generate_urls('alreadyexists/.osfkeep'): + aiohttpretty.register_uri('PUT', url, status=200) + + with pytest.raises(exceptions.FolderNamingConflict) as e: + await provider.create_folder(path) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_creates(self, provider, folder_metadata, mock_time): + path = WaterButlerPath('/doesntalreadyexists/') + for url in provider.generate_urls(secondary=True): + aiohttpretty.register_uri('GET', url, + params={'restype': 'container', 'comp': 'list'}, + body=folder_metadata, + headers={'Content-Type': 'application/xml'}) + for url in provider.generate_urls('doesntalreadyexists', secondary=True): + aiohttpretty.register_uri('HEAD', url, status=404) + for url in provider.generate_urls('doesntalreadyexists/.osfkeep'): + aiohttpretty.register_uri('PUT', url, status=200) + + resp = await provider.create_folder(path) + + assert resp.kind == 'folder' + assert resp.name == 'doesntalreadyexists' + assert resp.path == '/doesntalreadyexists/' + + +class TestOperations: + + async def test_equality(self, provider, mock_time): + assert provider.can_intra_copy(provider) + assert provider.can_intra_move(provider) diff --git a/waterbutler/providers/azureblobstorage/LICENSE b/waterbutler/providers/azureblobstorage/LICENSE new file mode 100644 index 000000000..55c6b3044 --- /dev/null +++ b/waterbutler/providers/azureblobstorage/LICENSE @@ -0,0 +1,203 @@ + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 National Institute of Informatics + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/waterbutler/providers/azureblobstorage/README.md b/waterbutler/providers/azureblobstorage/README.md new file mode 100644 index 000000000..04bd82781 --- /dev/null +++ b/waterbutler/providers/azureblobstorage/README.md @@ -0,0 +1,9 @@ +# WaterButler AzureBlobStorage provider: Custom WaterButler providers for OSF in Japan + +## License + +[Apache License Version 2.0](LICENSE) © 2017 National Institute of Informatics + +## Setting up AzureBlobStorage provider + +No settings for the provider. diff --git a/waterbutler/providers/azureblobstorage/__init__.py b/waterbutler/providers/azureblobstorage/__init__.py new file mode 100644 index 000000000..a00d0f1f8 --- /dev/null +++ b/waterbutler/providers/azureblobstorage/__init__.py @@ -0,0 +1 @@ +from .provider import AzureBlobStorageProvider # noqa diff --git a/waterbutler/providers/azureblobstorage/metadata.py b/waterbutler/providers/azureblobstorage/metadata.py new file mode 100644 index 000000000..6b8fb40f1 --- /dev/null +++ b/waterbutler/providers/azureblobstorage/metadata.py @@ -0,0 +1,93 @@ +import os + +from waterbutler.core import metadata + + +class AzureBlobStorageMetadata(metadata.BaseMetadata): + + @property + def provider(self): + return 'azureblobstorage' + + @property + def name(self): + return os.path.split(self.path)[1] + + @property + def created_utc(self): + return None + + +class AzureBlobStorageFileMetadataHeaders(AzureBlobStorageMetadata, metadata.BaseFileMetadata): + + def __init__(self, path, headers): + self._path = path + # Cast to dict to clone as the headers will + # be destroyed when the request leaves scope + super().__init__(headers) + + @property + def path(self): + return '/' + self._path + + @property + def size(self): + return self.raw.properties.content_length + + @property + def content_type(self): + return self.raw.properties.content_settings.content_type + + @property + def modified(self): + return self.raw.properties.last_modified.strftime('%Y-%m-%d %H:%M:%S') + + @property + def etag(self): + return self.raw.properties.content_settings.content_md5 + + @property + def extra(self): + return { + 'md5': self.raw.properties.content_settings.content_md5 + } + + +class AzureBlobStorageFileMetadata(AzureBlobStorageMetadata, metadata.BaseFileMetadata): + + @property + def path(self): + return '/' + self.raw.name + + @property + def size(self): + return int(self.raw.properties.content_length) + + @property + def modified(self): + return self.raw.properties.last_modified.strftime('%Y-%m-%d %H:%M:%S') + + @property + def content_type(self): + return self.raw.properties.content_settings.content_type + + @property + def etag(self): + return self.raw.properties.content_settings.content_md5 + + @property + def extra(self): + return { + 'md5': self.raw.properties.content_settings.content_md5 + } + + +class AzureBlobStorageFolderMetadata(AzureBlobStorageMetadata, metadata.BaseFolderMetadata): + + @property + def name(self): + return self.raw['prefix'].split('/')[-2] + + @property + def path(self): + return '/' + self.raw['prefix'] diff --git a/waterbutler/providers/azureblobstorage/provider.py b/waterbutler/providers/azureblobstorage/provider.py new file mode 100644 index 000000000..044042bbc --- /dev/null +++ b/waterbutler/providers/azureblobstorage/provider.py @@ -0,0 +1,378 @@ +import hashlib +import asyncio +import aiohttp +import functools +from urllib.parse import urlparse +from urllib.parse import quote as url_quote +import uuid + +from azure.storage.blob import BlockBlobService +from azure.storage._serialization import _add_date_header +from azure.storage.blob._serialization import _get_path +from azure.storage.blob._deserialization import ( + _convert_xml_to_blob_list, + _parse_blob +) +from azure.storage._constants import ( + X_MS_VERSION, + USER_AGENT_STRING, +) + +from waterbutler.core import streams +from waterbutler.core import provider +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath + +from waterbutler.providers.azureblobstorage.metadata import AzureBlobStorageFileMetadata +from waterbutler.providers.azureblobstorage.metadata import AzureBlobStorageFolderMetadata +from waterbutler.providers.azureblobstorage.metadata import AzureBlobStorageFileMetadataHeaders + + +class _Request(object): + + def __init__(self, method, url, params, headers): + self.method = method + urlo = urlparse(url) + self.path = urlo.path + self.host = urlo.netloc + self.query = params + self.headers = headers + self.body = None + + +class _ResponseBody(object): + + def __init__(self, resp, body): + self.headers = dict(map(lambda h: (h[0].lower(), h[1]), resp.headers.items())) + self.body = body + + +def _update_request(request): + # append addtional headers based on the service + request.headers['x-ms-version'] = X_MS_VERSION + request.headers['User-Agent'] = USER_AGENT_STRING + request.headers['x-ms-client-request-id'] = str(uuid.uuid1()) + + # If the host has a path component (ex local storage), move it + path = request.host.split('/', 1) + if len(path) == 2: + request.host = path[0] + request.path = '/{}{}'.format(path[1], request.path) + + # Encode and optionally add local storage prefix to path + request.path = url_quote(request.path, '/()$=\',~') + + +class AzureBlobStorageProvider(provider.BaseProvider): + """Provider for Azure Blob Storage cloud storage service. + """ + NAME = 'azureblobstorage' + + def __init__(self, auth, credentials, settings): + """ + :param dict auth: Not used + :param dict credentials: Dict containing `username`, `password` and `tenant_name` + :param dict settings: Dict containing `container` + """ + super().__init__(auth, credentials, settings) + + self.connection = BlockBlobService(account_name=credentials['account_name'], + account_key=credentials['account_key']) + + self.container = settings['container'] + + def _get_host_locations(self, primary=True, secondary=False): + locations = [] + if primary: + locations.append(self.connection.primary_endpoint) + if secondary: + locations.append(self.connection.secondary_endpoint) + return locations + + def generate_urls(self, blob_name=None, primary=True, secondary=False): + if blob_name is None: + path = _get_path(self.container) + else: + path = _get_path(self.container, blob_name) + hosts = self._get_host_locations(primary, secondary) + return list(map(lambda h: "https://" + h + path, hosts)) + + @provider.throttle() + async def make_signed_request(self, method, urls, *args, **kwargs): + kwargs['headers'] = self.build_headers(**kwargs.get('headers', {})) + retry = _retry = kwargs.pop('retry', 2) + range = kwargs.pop('range', None) + expects = kwargs.pop('expects', None) + throws = kwargs.pop('throws', exceptions.ProviderError) + if range: + kwargs['headers']['Range'] = self._build_range_header(range) + + if callable(urls): + urls = urls() + httpreq = _Request(method, urls[0], kwargs.get('params', {}), kwargs['headers']) + _update_request(httpreq) + _add_date_header(httpreq) + self.connection.authentication.sign_request(httpreq) + + target_url = 0 + while retry >= 0: + try: + response = await aiohttp.request(method, urls[target_url % len(urls)], *args, **kwargs) + if expects and response.status not in expects: + raise (await exceptions.exception_from_response(response, error=throws, **kwargs)) + return response + except throws as e: + if retry <= 0 or e.code not in self._retry_on: + raise + await asyncio.sleep((1 + _retry - retry) * 2) + retry -= 1 + target_url += 1 + + async def validate_v1_path(self, path, **kwargs): + if path == '/': + return WaterButlerPath(path) + + implicit_folder = path.endswith('/') + + assert path.startswith('/') + if implicit_folder: + resp = await self.make_signed_request( + 'GET', + functools.partial(self.generate_urls, secondary=True), + params={'restype': 'container', 'comp': 'list'}, + expects=(200, 404), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + if resp.status == 404: + raise exceptions.NotFoundError(str(path)) + objects = _convert_xml_to_blob_list(_ResponseBody(resp, respbody)) + if len(list(filter(lambda o: o.name.startswith(path[1:]), + objects))) == 0: + raise exceptions.NotFoundError(str(path)) + else: + resp = await self.make_signed_request( + 'HEAD', + functools.partial(self.generate_urls, path[1:], secondary=True), + expects=(200, 404), + throws=exceptions.MetadataError, + ) + await resp.release() + if resp.status == 404: + raise exceptions.NotFoundError(str(path)) + + return WaterButlerPath(path) + + async def validate_path(self, path, **kwargs): + return WaterButlerPath(path) + + def can_duplicate_names(self): + return True + + def can_intra_copy(self, dest_provider, path=None): + # Not supported + return False + + def can_intra_move(self, dest_provider, path=None): + # Not supported + return False + + async def intra_copy(self, dest_provider, source_path, dest_path): + # Not supported + raise NotImplementedError() + + async def download(self, path, accept_url=False, version=None, range=None, **kwargs): + """ + :param str path: Path to the key you want to download + :param dict \*\*kwargs: Additional arguments that are ignored + :rtype: :class:`waterbutler.core.streams.ResponseStreamReader` + :raises: :class:`waterbutler.core.exceptions.DownloadError` + """ + + if not path.is_file: + raise exceptions.DownloadError('No file specified for download', code=400) + + assert not path.path.startswith('/') + urls = functools.partial(self.generate_urls, path.path, secondary=True) + + resp = await self.make_signed_request( + 'GET', + urls, + expects=(200, ), + throws=exceptions.MetadataError, + ) + + return streams.ResponseStreamReader(resp) + + async def upload(self, stream, path, conflict='replace', **kwargs): + """Uploads the given stream to Azure Blob Storage + + :param waterbutler.core.streams.RequestWrapper stream: The stream to put to Azure Blob Storage + :param str path: The full path of the key to upload to/into + + :rtype: dict, bool + """ + + path, exists = await self.handle_name_conflict(path, conflict=conflict) + stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5)) + headers = {'Content-Length': str(stream.size), 'x-ms-blob-type': 'BlockBlob'} + + assert not path.path.startswith('/') + + resp = await self.make_signed_request( + 'PUT', + functools.partial(self.generate_urls, path.path), + data=stream, + headers=headers, + skip_auto_headers={'CONTENT-TYPE'}, + expects=(200, 201, 202, ), + throws=exceptions.UploadError, + ) + await resp.release() + + return (await self.metadata(path, **kwargs)), not exists + + async def delete(self, path, confirm_delete=0, **kwargs): + """Deletes the key at the specified path + + :param str path: The path of the key to delete + :param int confirm_delete: Must be 1 to confirm root folder delete + """ + + if path.is_root: + if not confirm_delete == 1: + raise exceptions.DeleteError( + 'confirm_delete=1 is required for deleting root provider folder', + code=400 + ) + + if path.is_file: + assert not path.path.startswith('/') + resp = await self.make_signed_request( + 'DELETE', + functools.partial(self.generate_urls, path.path), + expects=(200, 202, 204), + throws=exceptions.MetadataError, + ) + await resp.release() + else: + await self._delete_folder(path, **kwargs) + + async def _delete_folder(self, path, **kwargs): + resp = await self.make_signed_request( + 'GET', + functools.partial(self.generate_urls, secondary=True), + params={'restype': 'container', 'comp': 'list'}, + expects=(200, ), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + objects = _convert_xml_to_blob_list(_ResponseBody(resp, respbody)) + objects = list(map(lambda o: (o.name[len(path.path):], o), + filter(lambda o: o.name.startswith(path.path), + objects))) + if len(objects) == 0 and not path.is_root: + raise exceptions.DeleteError('Not found', code=404) + for name, blob in objects: + resp = await self.make_signed_request( + 'DELETE', + functools.partial(self.generate_urls, blob.name), + expects=(200, 202, 204, 404), + throws=exceptions.MetadataError, + ) + await resp.release() + + async def revisions(self, path, **kwargs): + """Get past versions of the requested key + + :param str path: The path to a key + :rtype list: + """ + return [] + + async def metadata(self, path, revision=None, **kwargs): + """Get Metadata about the requested file or folder + + :param WaterButlerPath path: The path to a key or folder + :rtype: dict or list + """ + if path.is_dir: + return (await self._metadata_folder(path)) + + return (await self._metadata_file(path, revision=revision)) + + async def create_folder(self, path, folder_precheck=True, **kwargs): + """ + :param str path: The path to create a folder at + """ + + WaterButlerPath.validate_folder(path) + + if folder_precheck: + if (await self.exists(path)): + raise exceptions.FolderNamingConflict(str(path)) + if (await self.exists(await self.validate_path('/' + path.path[:-1]))): + raise exceptions.FolderNamingConflict(str(path)) + + headers = {'x-ms-blob-type': 'BlockBlob'} + resp = await self.make_signed_request( + 'PUT', + functools.partial(self.generate_urls, path.path + '.osfkeep'), + data='', + headers=headers, + skip_auto_headers={'CONTENT-TYPE'}, + expects=(200, 201, 202, ), + throws=exceptions.CreateFolderError + ) + await resp.release() + + return AzureBlobStorageFolderMetadata({'prefix': path.path}) + + async def _metadata_file(self, path, revision=None): + if revision == 'Latest': + revision = None + assert not path.path.startswith('/') + resp = await self.make_signed_request( + 'HEAD', + functools.partial(self.generate_urls, path.path, secondary=True), + expects=(200, ), + throws=exceptions.MetadataError, + ) + await resp.release() + blob = _parse_blob(_ResponseBody(resp, b''), path.path, None) + return AzureBlobStorageFileMetadataHeaders(path.path, blob) + + async def _metadata_folder(self, path): + resp = await self.make_signed_request( + 'GET', + functools.partial(self.generate_urls, secondary=True), + params={'restype': 'container', 'comp': 'list'}, + expects=(200, ), + throws=exceptions.MetadataError, + ) + respbody = await resp.read() + objects = _convert_xml_to_blob_list(_ResponseBody(resp, respbody)) + objects = list(map(lambda o: (o.name[len(path.path):], o), + filter(lambda o: o.name.startswith(path.path), + objects))) + if len(objects) == 0 and not path.is_root: + raise exceptions.MetadataError('Not found', code=404) + + contents = list(filter(lambda o: '/' not in o[0], objects)) + prefixes = sorted(set(map(lambda o: path.path + o[0][:o[0].index('/') + 1], + filter(lambda o: '/' in o[0], objects)))) + + items = [ + AzureBlobStorageFolderMetadata({'prefix': item}) + for item in prefixes + ] + + for content_path, content in contents: + if content_path == path.path: + continue + fmetadata = AzureBlobStorageFileMetadata(content) + if fmetadata.name == '.osfkeep': + continue + items.append(fmetadata) + + return items diff --git a/waterbutler/providers/azureblobstorage/settings.py b/waterbutler/providers/azureblobstorage/settings.py new file mode 100644 index 000000000..43b28a2b7 --- /dev/null +++ b/waterbutler/providers/azureblobstorage/settings.py @@ -0,0 +1,3 @@ +from waterbutler import settings + +config = settings.child('AZUREBLOBSTORAGE_PROVIDER_CONFIG')