From 291eb4adbcccd42e259ba8842fd039aabf0aaffb Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 12 Oct 2017 15:39:44 -0400 Subject: [PATCH 01/15] Expand cloudfiles provider to preform all major functions, handle revisions etc. --- tests/providers/cloudfiles/fixtures.py | 243 ++++++++ .../cloudfiles/fixtures/fixtures.json | 181 ++++++ tests/providers/cloudfiles/test_metadata.py | 48 +- tests/providers/cloudfiles/test_provider.py | 568 +++++++----------- waterbutler/providers/cloudfiles/metadata.py | 35 +- waterbutler/providers/cloudfiles/provider.py | 181 ++++-- 6 files changed, 818 insertions(+), 438 deletions(-) create mode 100644 tests/providers/cloudfiles/fixtures.py create mode 100644 tests/providers/cloudfiles/fixtures/fixtures.json diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py new file mode 100644 index 000000000..c7704198d --- /dev/null +++ b/tests/providers/cloudfiles/fixtures.py @@ -0,0 +1,243 @@ +import os +import io +import time +import json +import pytest +import aiohttp +import aiohttpretty + +from unittest import mock + + +from waterbutler.core import streams +from waterbutler.providers.cloudfiles import CloudFilesProvider + + + +@pytest.fixture +def auth(): + return { + 'name': 'cat', + 'email': 'cat@cat.com', + } + + +@pytest.fixture +def credentials(): + return { + 'username': 'prince', + 'token': 'revolutionary', + 'region': 'iad', + } + + +@pytest.fixture +def settings(): + return {'container': 'purple rain'} + + +@pytest.fixture +def provider(auth, credentials, settings): + return CloudFilesProvider(auth, credentials, settings) + + + +@pytest.fixture +def token(auth_json): + return auth_json['access']['token']['id'] + + +@pytest.fixture +def endpoint(auth_json): + return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] + + +@pytest.fixture +def temp_url_key(): + return 'temporary beret' + + +@pytest.fixture +def mock_auth(auth_json): + aiohttpretty.register_json_uri( + 'POST', + settings.AUTH_URL, + body=auth_json, + ) + + +@pytest.fixture +def mock_temp_key(endpoint, temp_url_key): + aiohttpretty.register_uri( + 'HEAD', + endpoint, + status=204, + headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, + ) + + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def connected_provider(provider, token, endpoint, temp_url_key, mock_time): + provider.token = token + provider.endpoint = endpoint + provider.temp_url_key = temp_url_key.encode() + return provider + + +@pytest.fixture +def file_content(): + return b'sleepy' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + + +@pytest.fixture +def folder_root_empty(): + return [] + +@pytest.fixture +def container_header_metadata_with_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_with_verision_location'] + + +@pytest.fixture +def container_header_metadata_without_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_without_verision_location'] + +@pytest.fixture +def file_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_metadata'] + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + + +@pytest.fixture +def folder_root_level1_level2(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1_level2'] + + +@pytest.fixture +def folder_root_level1(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1'] + + +@pytest.fixture +def file_header_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_header_metadata'] + + +@pytest.fixture +def auth_json(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['auth_json'] + + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + +@pytest.fixture +def revision_list(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['revision_list'] + +@pytest.fixture +def file_root_level1_level2_file2_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) + + +@pytest.fixture +def folder_root_level1_empty(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '0'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), + ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), + ('X-TIMESTAMP', '1419274735.03160'), + ('CONTENT-TYPE', 'application/directory'), + ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') + ]) + + +@pytest.fixture +def file_root_similar(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419031343.23224'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), + ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') + ]) + + +@pytest.fixture +def file_root_similar_name(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419275231.66160'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), + ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') + ]) + + + +@pytest.fixture +def file_header_metadata_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) diff --git a/tests/providers/cloudfiles/fixtures/fixtures.json b/tests/providers/cloudfiles/fixtures/fixtures.json new file mode 100644 index 000000000..2e313777b --- /dev/null +++ b/tests/providers/cloudfiles/fixtures/fixtures.json @@ -0,0 +1,181 @@ +{ + "container_header_metadata_with_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3", + "X-VERSIONS-LOCATION":"versions-container" + }, + "container_header_metadata_without_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3" + }, + "file_metadata":{ + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + "file_header_metadata":{ + "CONTENT-LENGTH":"90", + "ACCEPT-RANGES":"bytes", + "LAST-MODIFIED":"Wed, 11 Oct 2017 21:37:55 GMT", + "ETAG":"8a839ea73aaa78718e27e025bdc2c767", + "X-TIMESTAMP":"1507757874.70544", + "CONTENT-TYPE":"application/octet-stream", + "X-TRANS-ID":"txae77ecf20a83452ebe2c0-0059dfa57aiad3", + "DATE":"Thu, 12 Oct 2017 17:25:14 GMT" + }, + "folder_root":[ + { + "last_modified":"2014-12-19T22:08:23.006360", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1", + "bytes":0 + }, + { + "subdir":"level1/" + }, + { + "last_modified":"2014-12-19T23:22:23.232240", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:20:16.718860", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1_empty", + "bytes":0 + } + ], + "folder_root_level1":[ + { + "last_modified":"2014-12-19T22:08:26.958830", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1/level2", + "bytes":0 + }, + { + "subdir":"level1/level2/" + } + ], + "folder_root_level1_level2":[ + { + "name":"level1/level2/file2.txt", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "last_modified":"2014-12-19T23:25:22.497420", + "bytes":1365336, + "hash":"ebc8cdd3f712fd39476fb921d43aca1a" + } + ], + "auth_json":{ + "access":{ + "serviceCatalog":[ + { + "name":"cloudFiles", + "type":"object-store", + "endpoints":[ + { + "publicURL":"https://fakestorage", + "internalURL":"https://internal_fake_storage", + "region":"IAD", + "tenantId":"someid_123456" + } + ] + } + ], + "token":{ + "RAX-AUTH:authenticatedBy":[ + "APIKEY" + ], + "tenant":{ + "name":"12345", + "id":"12345" + }, + "id":"2322f6b2322f4dbfa69802baf50b0832", + "expires":"2014-12-17T09:12:26.069Z" + }, + "user":{ + "name":"osf-production", + "roles":[ + { + "name":"object-store:admin", + "id":"10000256", + "description":"Object Store Admin Role for Account User" + }, + { + "name":"compute:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"6", + "tenantId":"12345" + }, + { + "name":"object-store:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"5", + "tenantId":"some_id_12345" + }, + { + "name":"identity:default", + "id":"2", + "description":"Default Role." + } + ], + "id":"secret", + "RAX-AUTH:defaultRegion":"IAD" + } + } + }, + "revision_list":[ + { + "hash":"8a839ea73aaa78718e27e025bdc2c767", + "bytes":90, + "name":"007123.csv/1507756317.92019", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:24:43.459520" + }, + { + "hash":"cacef99009078d6fbf994dd18aac5658", + "bytes":90, + "name":"007123.csv/1507757083.60055", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:31:34.386170" + }, + { + "hash":"63e4d56ff3b8a3bf4981f071dac1522e", + "bytes":90, + "name":"007123.csv/1507757494.53144", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:37:54.645380" + } + ] +} \ No newline at end of file diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index b69e44cc3..e304b3dce 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -1,36 +1,16 @@ import pytest -import aiohttp from waterbutler.core.path import WaterButlerPath from waterbutler.providers.cloudfiles.metadata import (CloudFilesFileMetadata, CloudFilesHeaderMetadata, - CloudFilesFolderMetadata) - - -@pytest.fixture -def file_header_metadata_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def file_metadata(): - return { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - } + CloudFilesFolderMetadata, + CloudFilesRevisonMetadata) + +from tests.providers.cloudfiles.fixtures import ( + file_header_metadata_txt, + file_metadata, + revision_list +) class TestCloudfilesMetadata: @@ -38,7 +18,7 @@ class TestCloudfilesMetadata: def test_header_metadata(self, file_header_metadata_txt): path = WaterButlerPath('/file.txt') - data = CloudFilesHeaderMetadata(file_header_metadata_txt, path.path) + data = CloudFilesHeaderMetadata(file_header_metadata_txt, path) assert data.name == 'file.txt' assert data.path == '/file.txt' assert data.provider == 'cloudfiles' @@ -242,3 +222,13 @@ def test_folder_metadata(self): 'new_folder': ('http://localhost:7777/v1/resources/' 'cn42d/providers/cloudfiles/level1/?kind=folder') } + + def test_revision_metadata(self, revision_list): + data = CloudFilesRevisonMetadata(revision_list[0]) + + assert data.version_identifier == 'revision' + assert data.name == '007123.csv/1507756317.92019' + assert data.version == '007123.csv/1507756317.92019' + assert data.size == 90 + assert data.content_type == 'application/octet-stream' + assert data.modified == '2017-10-11T21:24:43.459520' diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index da8bb55a5..a03e021b8 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -1,331 +1,48 @@ -import io -import os import json -import time import hashlib -import functools -from unittest import mock import furl import pytest import aiohttp import aiohttpretty -import aiohttp.multidict -from waterbutler.core import streams +from tests.utils import MockCoroutine from waterbutler.core import exceptions from waterbutler.core.path import WaterButlerPath -from waterbutler.providers.cloudfiles import CloudFilesProvider from waterbutler.providers.cloudfiles import settings as cloud_settings - -@pytest.fixture -def auth(): - return { - 'name': 'cat', - 'email': 'cat@cat.com', - } - - -@pytest.fixture -def credentials(): - return { - 'username': 'prince', - 'token': 'revolutionary', - 'region': 'iad', - } - - -@pytest.fixture -def settings(): - return {'container': 'purple rain'} - - -@pytest.fixture -def provider(auth, credentials, settings): - return CloudFilesProvider(auth, credentials, settings) - - -@pytest.fixture -def auth_json(): - return { - "access": { - "serviceCatalog": [ - { - "name": "cloudFiles", - "type": "object-store", - "endpoints": [ - { - "publicURL": "https://fakestorage", - "internalURL": "https://internal_fake_storage", - "region": "IAD", - "tenantId": "someid_123456" - }, - ] - } - ], - "token": { - "RAX-AUTH:authenticatedBy": [ - "APIKEY" - ], - "tenant": { - "name": "12345", - "id": "12345" - }, - "id": "2322f6b2322f4dbfa69802baf50b0832", - "expires": "2014-12-17T09:12:26.069Z" - }, - "user": { - "name": "osf-production", - "roles": [ - { - "name": "object-store:admin", - "id": "10000256", - "description": "Object Store Admin Role for Account User" - }, - { - "name": "compute:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "6", - "tenantId": "12345" - }, - { - "name": "object-store:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "5", - "tenantId": "some_id_12345" - }, - { - "name": "identity:default", - "id": "2", - "description": "Default Role." - } - ], - "id": "secret", - "RAX-AUTH:defaultRegion": "IAD" - } - } - } - - -@pytest.fixture -def token(auth_json): - return auth_json['access']['token']['id'] - - -@pytest.fixture -def endpoint(auth_json): - return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] - - -@pytest.fixture -def temp_url_key(): - return 'temporary beret' - - -@pytest.fixture -def mock_auth(auth_json): - aiohttpretty.register_json_uri( - 'POST', - settings.AUTH_URL, - body=auth_json, - ) - - -@pytest.fixture -def mock_temp_key(endpoint, temp_url_key): - aiohttpretty.register_uri( - 'HEAD', - endpoint, - status=204, - headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, - ) - - -@pytest.fixture -def mock_time(monkeypatch): - mock_time = mock.Mock() - mock_time.return_value = 10 - monkeypatch.setattr(time, 'time', mock_time) - - -@pytest.fixture -def connected_provider(provider, token, endpoint, temp_url_key, mock_time): - provider.token = token - provider.endpoint = endpoint - provider.temp_url_key = temp_url_key.encode() - return provider - - -@pytest.fixture -def file_content(): - return b'sleepy' - - -@pytest.fixture -def file_like(file_content): - return io.BytesIO(file_content) - - -@pytest.fixture -def file_stream(file_like): - return streams.FileStreamReader(file_like) - - -@pytest.fixture -def file_metadata(): - return aiohttp.multidict.CIMultiDict([ - ('LAST-MODIFIED', 'Thu, 25 Dec 2014 02:54:35 GMT'), - ('CONTENT-LENGTH', '0'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('CONTENT-TYPE', 'text/html; charset=UTF-8'), - ('X-TRANS-ID', 'txf876a4b088e3451d94442-00549b7c6aiad3'), - ('DATE', 'Thu, 25 Dec 2014 02:54:34 GMT') - ]) - - -# Metadata Test Scenarios -# / (folder_root_empty) -# / (folder_root) -# /level1/ (folder_root_level1) -# /level1/level2/ (folder_root_level1_level2) -# /level1/level2/file2.file - (file_root_level1_level2_file2_txt) -# /level1_empty/ (folder_root_level1_empty) -# /similar (file_similar) -# /similar.name (file_similar_name) -# /does_not_exist (404) -# /does_not_exist/ (404) - - -@pytest.fixture -def folder_root_empty(): - return [] - - -@pytest.fixture -def folder_root(): - return [ - { - 'last_modified': '2014-12-19T22:08:23.006360', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1', - 'bytes': 0 - }, - { - 'subdir': 'level1/' - }, - { - 'last_modified': '2014-12-19T23:22:23.232240', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:20:16.718860', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1_empty', - 'bytes': 0 - } - ] - - -@pytest.fixture -def folder_root_level1(): - return [ - { - 'last_modified': '2014-12-19T22:08:26.958830', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1/level2', - 'bytes': 0 - }, - { - 'subdir': 'level1/level2/' - } - ] - - -@pytest.fixture -def folder_root_level1_level2(): - return [ - { - 'name': 'level1/level2/file2.txt', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'last_modified': '2014-12-19T23:25:22.497420', - 'bytes': 1365336, - 'hash': 'ebc8cdd3f712fd39476fb921d43aca1a' - } - ] - - -@pytest.fixture -def file_root_level1_level2_file2_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def folder_root_level1_empty(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '0'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), - ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), - ('X-TIMESTAMP', '1419274735.03160'), - ('CONTENT-TYPE', 'application/directory'), - ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') - ]) - - -@pytest.fixture -def file_root_similar(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419031343.23224'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), - ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') - ]) - - -@pytest.fixture -def file_root_similar_name(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419275231.66160'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), - ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') - ]) +from waterbutler.providers.cloudfiles.metadata import (CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata) + +from tests.providers.cloudfiles.fixtures import ( + auth, + settings, + credentials, + token, + endpoint, + temp_url_key, + mock_time, + file_content, + file_stream, + file_like, + mock_temp_key, + provider, + connected_provider, + file_metadata, + auth_json, + folder_root, + folder_root_empty, + folder_root_level1_empty, + file_root_similar_name, + file_root_similar, + file_header_metadata, + folder_root_level1, + folder_root_level1_level2, + file_root_level1_level2_file2_txt, + container_header_metadata_with_verision_location, + container_header_metadata_without_verision_location, + revision_list +) class TestCRUD: @@ -344,6 +61,31 @@ async def test_download(self, connected_provider): assert content == body + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download_revision(self, + connected_provider, + container_header_metadata_with_verision_location): + body = b'dearly-beloved' + path = WaterButlerPath('/lets-go-crazy') + url = connected_provider.sign_url(path) + aiohttpretty.register_uri('GET', url, body=body, auto_length=True) + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + revision_url = connected_provider.build_url(version_name, container='versions-container', ) + aiohttpretty.register_uri('GET', revision_url, body=body, auto_length=True) + + + result = await connected_provider.download(path, version=version_name) + content = await result.read() + + assert content == body + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_download_accept_url(self, connected_provider): @@ -373,21 +115,26 @@ async def test_download_not_found(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload(self, connected_provider, file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload(self, + connected_provider, + file_content, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_header_metadata} + ] + ) + aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"{}"'.format(content_md5)}) + metadata, created = await connected_provider.upload(file_stream, path) assert created is True @@ -395,22 +142,31 @@ async def test_upload(self, connected_provider, file_content, file_stream, file_ assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_create_folder(self, connected_provider, file_header_metadata): + path = WaterButlerPath('/foo/', folder=True) + metadata_url = connected_provider.build_url(path.path) + url = connected_provider.sign_url(path, 'PUT') + print(metadata_url) + aiohttpretty.register_uri('PUT', url, status=200) + aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) + + metadata = await connected_provider.create_folder(path) + + assert metadata.kind == 'folder' + assert aiohttpretty.has_call(method='PUT', uri=url) + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_upload_check_none(self, connected_provider, - file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + file_content, file_stream, file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"{}"'.format(content_md5)}) metadata, created = await connected_provider.upload( @@ -422,18 +178,15 @@ async def test_upload_check_none(self, connected_provider, @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload_checksum_mismatch(self, connected_provider, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload_checksum_mismatch(self, + connected_provider, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) + aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"Bogus MD5"'}) with pytest.raises(exceptions.UploadChecksumMismatchError): @@ -442,30 +195,30 @@ async def test_upload_checksum_mismatch(self, connected_provider, file_stream, f assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) - # @pytest.mark.asyncio - # @pytest.mark.aiohttpretty - # async def test_delete_folder(self, connected_provider, folder_root_empty, file_metadata): - # # This test will probably fail on a live - # # version of the provider because build_url is called wrong. - # # Will comment out parts of this test till that is fixed. - # path = WaterButlerPath('/delete/') - # query = {'prefix': path.path} - # url = connected_provider.build_url('', **query) - # body = json.dumps(folder_root_empty).encode('utf-8') + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_folder(self, connected_provider, folder_root_empty, file_header_metadata): - # delete_query = {'bulk-delete': ''} - # delete_url = connected_provider.build_url('', **delete_query) + path = WaterButlerPath('/delete/') + query = {'prefix': path.path} + url = connected_provider.build_url('', **query) + body = json.dumps(folder_root_empty).encode('utf-8') - # file_url = connected_provider.build_url(path.path) + delete_query = {'bulk-delete': ''} + delete_url_folder = connected_provider.build_url(path.name, **delete_query) + delete_url_content = connected_provider.build_url('', **delete_query) - # aiohttpretty.register_uri('GET', url, body=body) - # aiohttpretty.register_uri('HEAD', file_url, headers=file_metadata) + file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('GET', url, body=body) + aiohttpretty.register_uri('HEAD', file_url, headers=file_header_metadata) - # aiohttpretty.register_uri('DELETE', delete_url) + aiohttpretty.register_uri('DELETE', delete_url_content, status=200) + aiohttpretty.register_uri('DELETE', delete_url_folder, status=204) - # await connected_provider.delete(path) + await connected_provider.delete(path) - # assert aiohttpretty.has_call(method='DELETE', uri=delete_url) + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_folder) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -479,21 +232,74 @@ async def test_delete_file(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_intra_copy(self, connected_provider, file_metadata): + async def test_intra_copy(self, connected_provider, file_header_metadata): src_path = WaterButlerPath('/delete.file') dest_path = WaterButlerPath('/folder1/delete.file') dest_url = connected_provider.build_url(dest_path.path) - aiohttpretty.register_uri('HEAD', dest_url, headers=file_metadata) + aiohttpretty.register_uri('HEAD', dest_url, headers=file_header_metadata) aiohttpretty.register_uri('PUT', dest_url, status=201) result = await connected_provider.intra_copy(connected_provider, src_path, dest_path) assert result[0].path == '/folder1/delete.file' assert result[0].name == 'delete.file' - assert result[0].etag == 'edfa12d00b779b4b37b81fe5b61b2b3f' + assert result[0].etag == '8a839ea73aaa78718e27e025bdc2c767' + + +class TestRevisions: + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions(self, + connected_provider, + container_header_metadata_with_verision_location, + revision_list): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + revision_url = connected_provider.build_url('', container='versions-container', **query) + aiohttpretty.register_json_uri('GET', revision_url, body=revision_list) + + result = await connected_provider.revisions(path) + + assert type(result) == list + assert len(result) == 3 + assert type(result[0]) == CloudFilesRevisonMetadata + assert result[0].name == '007123.csv/1507756317.92019' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revision_metadata(self, + connected_provider, + container_header_metadata_with_verision_location, + file_header_metadata): + + path = WaterButlerPath('/file.txt') + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + query = {'prefix' : version_name} + revision_url = connected_provider.build_url(version_name + '1507756317.92019', + container='versions-container') + print(revision_url) + aiohttpretty.register_json_uri('HEAD', revision_url, body=file_header_metadata) + + result = await connected_provider.metadata(path, version=version_name + '1507756317.92019') + + assert type(result) == CloudFilesHeaderMetadata + assert result.name == 'file.txt' + assert result.path == '/file.txt' + assert result.kind == 'file' class TestMetadata: @@ -532,6 +338,18 @@ async def test_metadata_folder_root(self, connected_provider, folder_root): assert result[3].path == '/level1_empty/' assert result[3].kind == 'folder' + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_404(self, connected_provider, folder_root_level1): + path = WaterButlerPath('/level1/') + body = json.dumps(folder_root_level1).encode('utf-8') + url = connected_provider.build_url('', prefix=path.path, delimiter='/') + aiohttpretty.register_uri('GET', url, status=200, body=b'') + connected_provider._metadata_item = MockCoroutine(return_value=None) + + with pytest.raises(exceptions.MetadataError): + await connected_provider.metadata(path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_root_level1(self, connected_provider, folder_root_level1): @@ -674,6 +492,22 @@ async def test_ensure_connection(self, provider, auth_json, mock_temp_key): await provider._ensure_connection() assert aiohttpretty.has_call(method='POST', uri=token_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_no_version_location(self, + connected_provider, + container_header_metadata_without_verision_location): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_without_verision_location) + + with pytest.raises(exceptions.MetadataError): + await connected_provider.revisions(path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_ensure_connection_not_public(self, provider, auth_json, temp_url_key): diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index aa13c1ed9..f13f192ca 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -55,13 +55,17 @@ def __init__(self, raw, path): super().__init__(raw) self._path = path + @property + def kind(self): + return 'folder' if self._path.is_dir else 'file' + @property def name(self): - return os.path.split(self._path)[1] + return self._path.name @property def path(self): - return self.build_path(self._path) + return self._path.materialized_path @property def size(self): @@ -101,3 +105,30 @@ def name(self): @property def path(self): return self.build_path(self.raw['subdir']) + + +class CloudFilesRevisonMetadata(metadata.BaseFileRevisionMetadata): + + @property + def version_identifier(self): + return 'revision' + + @property + def version(self): + return self.raw['name'] + + @property + def modified(self): + return self.raw['last_modified'] + + @property + def size(self): + return self.raw['bytes'] + + @property + def name(self): + return self.raw['name'] + + @property + def content_type(self): + return self.raw['content_type'] diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index fa0248571..4b47fd455 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -14,9 +14,12 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.providers.cloudfiles import settings -from waterbutler.providers.cloudfiles.metadata import CloudFilesFileMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesFolderMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesHeaderMetadata +from waterbutler.providers.cloudfiles.metadata import ( + CloudFilesFileMetadata, + CloudFilesFolderMetadata, + CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata +) def ensure_connection(func): @@ -32,7 +35,8 @@ async def wrapped(self, *args, **kwargs): class CloudFilesProvider(provider.BaseProvider): """Provider for Rackspace CloudFiles. - API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/#document-developer-guide + API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/ + #document-developer-guide """ NAME = 'cloudfiles' @@ -79,7 +83,14 @@ async def intra_copy(self, dest_provider, source_path, dest_path): return (await dest_provider.metadata(dest_path)), not exists @ensure_connection - async def download(self, path, accept_url=False, range=None, **kwargs): + async def download(self, + path, + accept_url=False, + range=None, + version=None, + revision=None, + displayName=None, + **kwargs): """Returns a ResponseStreamReader (Stream) for the specified path :param str path: Path to the object you want to download :param dict \*\*kwargs: Additional arguments that are ignored @@ -90,9 +101,13 @@ async def download(self, path, accept_url=False, range=None, **kwargs): self.metrics.add('download.accept_url', accept_url) if accept_url: parsed_url = furl.furl(self.sign_url(path, endpoint=self.public_endpoint)) - parsed_url.args['filename'] = kwargs.get('displayName') or path.name + parsed_url.args['filename'] = displayName or path.name return parsed_url.url + version = revision or version + if version: + return await self._download_revision(range, version) + resp = await self.make_request( 'GET', functools.partial(self.sign_url, path), @@ -103,11 +118,13 @@ async def download(self, path, accept_url=False, range=None, **kwargs): return streams.ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True, **kwargs): + async def upload(self, stream, path, check_created=True, fetch_metadata=True): """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into - :rtype ResponseStreamReader: + :param bool check_created: This checks if uploaded file already exists + :param bool fetch_metadata: If true upload will return metadata + :rtype (dict/None, bool): """ if check_created: created = not (await self.exists(path)) @@ -138,25 +155,25 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** return metadata, created @ensure_connection - async def delete(self, path, **kwargs): + async def delete(self, path, confirm_delete=False): """Deletes the key at the specified path :param str path: The path of the key to delete - :rtype ResponseStreamReader: + :param bool confirm_delete: not used in this provider + :rtype None: """ if path.is_dir: - metadata = await self.metadata(path, recursive=True) + metadata = await self._metadata_folder(path, recursive=True) delete_files = [ - os.path.join('/', self.container, path.child(item['name']).path) + os.path.join('/', self.container, path.child(item.name).path) for item in metadata ] delete_files.append(os.path.join('/', self.container, path.path)) - query = {'bulk-delete': ''} resp = await self.make_request( 'DELETE', - functools.partial(self.build_url, **query), + functools.partial(self.build_url, '', **query), data='\n'.join(delete_files), expects=(200, ), throws=exceptions.DeleteError, @@ -164,35 +181,39 @@ async def delete(self, path, **kwargs): 'Content-Type': 'text/plain', }, ) - else: - resp = await self.make_request( - 'DELETE', - functools.partial(self.build_url, path.path), - expects=(204, ), - throws=exceptions.DeleteError, - ) + await resp.release() + + resp = await self.make_request( + 'DELETE', + functools.partial(self.build_url, path.path), + expects=(204, ), + throws=exceptions.DeleteError, + ) await resp.release() @ensure_connection - async def metadata(self, path, recursive=False, **kwargs): + async def metadata(self, path, recursive=False, version=None, revision=None): """Get Metadata about the requested file or folder :param str path: The path to a key or folder :rtype dict: :rtype list: """ if path.is_dir: - return (await self._metadata_folder(path, recursive=recursive, **kwargs)) + return (await self._metadata_folder(path, recursive=recursive)) + elif version or revision: + return (await self.get_metadata_revision(path, version, revision)) else: - return (await self._metadata_file(path, **kwargs)) + return (await self._metadata_item(path)) - def build_url(self, path, _endpoint=None, **query): + def build_url(self, path, _endpoint=None, container=None, **query): """Build the url for the specified object :param args segments: URI segments :param kwargs query: Query parameters :rtype str: """ endpoint = _endpoint or self.endpoint - return provider.build_url(endpoint, self.container, *path.split('/'), **query) + container = container or self.container + return provider.build_url(endpoint, container, *path.split('/'), **query) def can_duplicate_names(self): return False @@ -211,18 +232,23 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ :param int seconds: Time for the url to live :rtype str: """ + from urllib.parse import unquote + method = method.upper() expires = str(int(time.time() + seconds)) - url = furl.furl(self.build_url(path.path, _endpoint=endpoint)) + path_str = path.path + url = furl.furl(self.build_url(path_str, _endpoint=endpoint)) - body = '\n'.join([method, expires, str(url.path)]).encode() + body = '\n'.join([method, expires]) + body += '\n' + str(url.path) + body = unquote(body).encode() signature = hmac.new(self.temp_url_key, body, hashlib.sha1).hexdigest() - url.args.update({ 'temp_url_sig': signature, 'temp_url_expires': expires, }) - return url.url + + return unquote(str(url.url)) async def make_request(self, *args, **kwargs): try: @@ -297,7 +323,7 @@ async def _get_token(self): data = await resp.json() return data - async def _metadata_file(self, path, is_folder=False, **kwargs): + async def _metadata_item(self, path): """Get Metadata about the requested file :param str path: The path to a key :rtype dict: @@ -309,18 +335,17 @@ async def _metadata_file(self, path, is_folder=False, **kwargs): expects=(200, ), throws=exceptions.MetadataError, ) - await resp.release() - if (resp.headers['Content-Type'] == 'application/directory' and not is_folder): + if resp.headers['Content-Type'] == 'application/directory' and path.is_file: raise exceptions.MetadataError( 'Could not retrieve file \'{0}\''.format(str(path)), code=404, ) - return CloudFilesHeaderMetadata(resp.headers, path.path) + return CloudFilesHeaderMetadata(resp.headers, path) - async def _metadata_folder(self, path, recursive=False, **kwargs): + async def _metadata_folder(self, path, recursive=False): """Get Metadata about the requested folder :param str path: The path to a folder :rtype dict: @@ -339,11 +364,12 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): ) data = await resp.json() - # no data and the provider path is not root, we are left with either a file or a directory marker + # no data and the provider path is not root, we are left with either a file or a directory + # marker if not data and not path.is_root: # Convert the parent path into a directory marker (file) and check for an empty folder - dir_marker = path.parent.child(path.name, folder=False) - metadata = await self._metadata_file(dir_marker, is_folder=True, **kwargs) + dir_marker = path.parent.child(path.name, folder=path.is_dir) + metadata = await self._metadata_item(dir_marker) if not metadata: raise exceptions.MetadataError( 'Could not retrieve folder \'{0}\''.format(str(path)), @@ -361,13 +387,88 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): break return [ - self._serialize_folder_metadata(item) + self._serialize_metadata(item) for item in data ] - def _serialize_folder_metadata(self, data): + def _serialize_metadata(self, data): if data.get('subdir'): return CloudFilesFolderMetadata(data) elif data['content_type'] == 'application/directory': return CloudFilesFolderMetadata({'subdir': data['name'] + '/'}) return CloudFilesFileMetadata(data) + + @ensure_connection + async def create_folder(self, path): + + resp = await self.make_request( + 'PUT', + functools.partial(self.sign_url, path, 'PUT'), + expects=(200, 201), + throws=exceptions.UploadError, + headers={'Content-Type': 'application/directory'} + ) + await resp.release() + return await self._metadata_item(path) + + @ensure_connection + async def _get_version_location(self): + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, ''), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + await resp.release() + + try: + return resp.headers['X-VERSIONS-LOCATION'] + except KeyError: + raise exceptions.MetadataError('The your container does not have a defined version' + ' location. To set a version location and store file ' + 'versions follow the instructions here: ' + 'https://developer.rackspace.com/docs/cloud-files/v1/' + 'use-cases/additional-object-services-information/' + '#object-versioning') + + @ensure_connection + async def revisions(self, path): + version_location = await self._get_version_location() + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, '', container=version_location, **query), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + json_resp = await resp.json() + return [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] + + @ensure_connection + async def get_metadata_revision(self, path, version=None, revision=None): + version_location = await self._get_version_location() + + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, version or revision, container=version_location), + expects=(200, ), + throws=exceptions.MetadataError, + ) + await resp.release() + + return CloudFilesHeaderMetadata(resp.headers, path) + + async def _download_revision(self, range, version): + + version_location = await self._get_version_location() + + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, version, container=version_location), + range=range, + expects=(200, 206), + throws=exceptions.DownloadError + ) + + return streams.ResponseStreamReader(resp) From 7a759d03f14830816f708195c8541eeb1d611741 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 13 Oct 2017 12:20:23 -0400 Subject: [PATCH 02/15] Add new revisions behavior and clean up metadata --- tests/providers/cloudfiles/test_provider.py | 15 +++++++++------ waterbutler/core/provider.py | 1 - waterbutler/providers/cloudfiles/metadata.py | 18 +++++++++++++----- waterbutler/providers/cloudfiles/provider.py | 7 ++++--- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index a03e021b8..eaad6a6fa 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -148,7 +148,6 @@ async def test_create_folder(self, connected_provider, file_header_metadata): path = WaterButlerPath('/foo/', folder=True) metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - print(metadata_url) aiohttpretty.register_uri('PUT', url, status=200) aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) @@ -255,7 +254,8 @@ class TestRevisions: async def test_revisions(self, connected_provider, container_header_metadata_with_verision_location, - revision_list): + revision_list, + file_header_metadata): path = WaterButlerPath('/file.txt') @@ -268,12 +268,17 @@ async def test_revisions(self, revision_url = connected_provider.build_url('', container='versions-container', **query) aiohttpretty.register_json_uri('GET', revision_url, body=revision_list) + metadata_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('HEAD', metadata_url, status=200, headers=file_header_metadata) + + result = await connected_provider.revisions(path) assert type(result) == list - assert len(result) == 3 + assert len(result) == 4 assert type(result[0]) == CloudFilesRevisonMetadata - assert result[0].name == '007123.csv/1507756317.92019' + assert result[0].name == 'file.txt' + assert result[1].name == '007123.csv/1507756317.92019' @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -288,10 +293,8 @@ async def test_revision_metadata(self, headers=container_header_metadata_with_verision_location) version_name = '{:03x}'.format(len(path.name)) + path.name + '/' - query = {'prefix' : version_name} revision_url = connected_provider.build_url(version_name + '1507756317.92019', container='versions-container') - print(revision_url) aiohttpretty.register_json_uri('HEAD', revision_url, body=file_header_metadata) result = await connected_provider.metadata(path, version=version_name + '1507756317.92019') diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index ba443e33f..559f9a46a 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -323,7 +323,6 @@ async def _folder_file_op(self, """ assert src_path.is_dir, 'src_path must be a directory' assert asyncio.iscoroutinefunction(func), 'func must be a coroutine' - try: await dest_provider.delete(dest_path) created = False diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index f13f192ca..5e44720b1 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -55,6 +55,14 @@ def __init__(self, raw, path): super().__init__(raw) self._path = path + def to_revision(self): + revison_dict = {'bytes': self.size, + 'name': self.name, + 'last_modified': self.modified, + 'content_type': self.content_type} + + return CloudFilesRevisonMetadata(revison_dict) + @property def kind(self): return 'folder' if self._path.is_dir else 'file' @@ -69,11 +77,11 @@ def path(self): @property def size(self): - return int(self.raw['Content-Length']) + return int(self.raw['CONTENT-LENGTH']) @property def modified(self): - return self.raw['Last-Modified'] + return self.raw['LAST-MODIFIED'] @property def created_utc(self): @@ -81,17 +89,17 @@ def created_utc(self): @property def content_type(self): - return self.raw['Content-Type'] + return self.raw['CONTENT-TYPE'] @property def etag(self): - return self.raw['etag'] + return self.raw['ETAG'] @property def extra(self): return { 'hashes': { - 'md5': self.raw['etag'].replace('"', ''), + 'md5': self.raw['ETAG'], }, } diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 4b47fd455..412468172 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -343,7 +343,7 @@ async def _metadata_item(self, path): code=404, ) - return CloudFilesHeaderMetadata(resp.headers, path) + return CloudFilesHeaderMetadata(dict(resp.headers), path) async def _metadata_folder(self, path, recursive=False): """Get Metadata about the requested folder @@ -399,7 +399,7 @@ def _serialize_metadata(self, data): return CloudFilesFileMetadata(data) @ensure_connection - async def create_folder(self, path): + async def create_folder(self, path, folder_precheck=None): resp = await self.make_request( 'PUT', @@ -443,7 +443,8 @@ async def revisions(self, path): throws=exceptions.MetadataError, ) json_resp = await resp.json() - return [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] + current = (await self.metadata(path)).to_revision() + return [current] + [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] @ensure_connection async def get_metadata_revision(self, path, version=None, revision=None): From 9fef6c683d5c8e8f315b3d8af7c9046a94549a84 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Tue, 17 Oct 2017 14:22:04 -0400 Subject: [PATCH 03/15] Allow all kwargs for registrations. --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 412468172..d189e3c55 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -192,7 +192,7 @@ async def delete(self, path, confirm_delete=False): await resp.release() @ensure_connection - async def metadata(self, path, recursive=False, version=None, revision=None): + async def metadata(self, path, recursive=False, version=None, revision=None, **kwargs): """Get Metadata about the requested file or folder :param str path: The path to a key or folder :rtype dict: From 03442e3eaecb10cb5155fe6a65fae6ea72af9f80 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:05:20 -0400 Subject: [PATCH 04/15] fix import order --- tests/providers/cloudfiles/fixtures.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py index c7704198d..deb6148f3 100644 --- a/tests/providers/cloudfiles/fixtures.py +++ b/tests/providers/cloudfiles/fixtures.py @@ -2,13 +2,12 @@ import io import time import json +from unittest import mock + import pytest import aiohttp import aiohttpretty -from unittest import mock - - from waterbutler.core import streams from waterbutler.providers.cloudfiles import CloudFilesProvider From a596fa59b2f1df821c30c7385368b51b9a97ae97 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:37:35 -0400 Subject: [PATCH 05/15] Add doc strings and clean up --- waterbutler/providers/cloudfiles/provider.py | 41 +++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index d189e3c55..3720c191f 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -5,6 +5,7 @@ import asyncio import hashlib import functools +from urllib.parse import unquote import furl @@ -35,8 +36,7 @@ async def wrapped(self, *args, **kwargs): class CloudFilesProvider(provider.BaseProvider): """Provider for Rackspace CloudFiles. - API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/ - #document-developer-guide + API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/document-developer-guide """ NAME = 'cloudfiles' @@ -158,9 +158,18 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): async def delete(self, path, confirm_delete=False): """Deletes the key at the specified path :param str path: The path of the key to delete - :param bool confirm_delete: not used in this provider + :param int confirm_delete: Must be 1 to confirm root folder delete, this deletes entire + container object. :rtype None: """ + + if path.is_root and not confirm_delete: + raise exceptions.DeleteError( + 'query arguement confirm_delete=1 is required for deleting the entire container.', + code=400 + ) + + if path.is_dir: metadata = await self._metadata_folder(path, recursive=True) @@ -232,7 +241,6 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ :param int seconds: Time for the url to live :rtype str: """ - from urllib.parse import unquote method = method.upper() expires = str(int(time.time() + seconds)) @@ -399,13 +407,22 @@ def _serialize_metadata(self, data): return CloudFilesFileMetadata(data) @ensure_connection - async def create_folder(self, path, folder_precheck=None): + async def create_folder(self, path, **kwargs): + """Create a folder in the current provider at `path`. Returns a `BaseFolderMetadata` object + if successful. May throw a 409 Conflict if a directory with the same name already exists. + Enpoint information can be found here: + https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/pseudo-hierarchical-folders-and-directories/ + :param path: ( :class:`.WaterButlerPath` )User-supplied path to create. Must be a directory. + :param kwargs: dict unused params + :rtype: :class:`.BaseFileMetadata` + :raises: :class:`.CreateFolderError` + """ resp = await self.make_request( 'PUT', functools.partial(self.sign_url, path, 'PUT'), expects=(200, 201), - throws=exceptions.UploadError, + throws=exceptions.CreateFolderError, headers={'Content-Type': 'application/directory'} ) await resp.release() @@ -433,6 +450,18 @@ async def _get_version_location(self): @ensure_connection async def revisions(self, path): + """Get past versions of the request file from special user designated version container, + if the user hasn't designated a version_location container in raises an infomative error + message. The revision endpoint also doesn't return the current version so that is added to + the revision list after other revisions are returned. More info about versioning with Cloud + Files here: + + https://developer.rackspace.com/docs/cloud-files/v1/use-cases/additional-object-services-information/#object-versioning + + :param str path: The path to a key + :rtype list: + """ + version_location = await self._get_version_location() query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} From d27bd747984b4dd76f4de61c22479a0253d3638c Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:42:36 -0400 Subject: [PATCH 06/15] flake fix --- waterbutler/providers/cloudfiles/provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 3720c191f..e0bbb2e22 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -169,7 +169,6 @@ async def delete(self, path, confirm_delete=False): code=400 ) - if path.is_dir: metadata = await self._metadata_folder(path, recursive=True) From 87e5fca24cc7d454b123175482a541e943b21e7c Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:52:50 -0400 Subject: [PATCH 07/15] fix misspelling --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index e0bbb2e22..0e669d25f 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -165,7 +165,7 @@ async def delete(self, path, confirm_delete=False): if path.is_root and not confirm_delete: raise exceptions.DeleteError( - 'query arguement confirm_delete=1 is required for deleting the entire container.', + 'query argument confirm_delete=1 is required for deleting the entire container.', code=400 ) From b78cffad5b0365491dbdc2f2c8f3088681e19acf Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 16 Nov 2017 09:42:22 -0500 Subject: [PATCH 08/15] clean up --- tests/providers/cloudfiles/test_provider.py | 7 +++---- waterbutler/providers/cloudfiles/provider.py | 11 +++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index eaad6a6fa..58c66d71a 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -200,7 +200,7 @@ async def test_delete_folder(self, connected_provider, folder_root_empty, file_h path = WaterButlerPath('/delete/') query = {'prefix': path.path} - url = connected_provider.build_url('', **query) + url = connected_provider.build_url('', **query) body = json.dumps(folder_root_empty).encode('utf-8') delete_query = {'bulk-delete': ''} @@ -271,7 +271,6 @@ async def test_revisions(self, metadata_url = connected_provider.build_url(path.path) aiohttpretty.register_uri('HEAD', metadata_url, status=200, headers=file_header_metadata) - result = await connected_provider.revisions(path) assert type(result) == list @@ -304,6 +303,7 @@ async def test_revision_metadata(self, assert result.path == '/file.txt' assert result.kind == 'file' + class TestMetadata: @pytest.mark.asyncio @@ -343,9 +343,8 @@ async def test_metadata_folder_root(self, connected_provider, folder_root): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_metadata_404(self, connected_provider, folder_root_level1): + async def test_metadata_404(self, connected_provider): path = WaterButlerPath('/level1/') - body = json.dumps(folder_root_level1).encode('utf-8') url = connected_provider.build_url('', prefix=path.path, delimiter='/') aiohttpretty.register_uri('GET', url, status=200, body=b'') connected_provider._metadata_item = MockCoroutine(return_value=None) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 0e669d25f..68c1da020 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -36,7 +36,7 @@ async def wrapped(self, *args, **kwargs): class CloudFilesProvider(provider.BaseProvider): """Provider for Rackspace CloudFiles. - API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/document-developer-guide + API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/#document-developer-guide """ NAME = 'cloudfiles' @@ -209,7 +209,7 @@ async def metadata(self, path, recursive=False, version=None, revision=None, **k if path.is_dir: return (await self._metadata_folder(path, recursive=recursive)) elif version or revision: - return (await self.get_metadata_revision(path, version, revision)) + return (await self._metadata_revision(path, version, revision)) else: return (await self._metadata_item(path)) @@ -243,8 +243,7 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ method = method.upper() expires = str(int(time.time() + seconds)) - path_str = path.path - url = furl.furl(self.build_url(path_str, _endpoint=endpoint)) + url = furl.furl(self.build_url(path.path, _endpoint=endpoint)) body = '\n'.join([method, expires]) body += '\n' + str(url.path) @@ -450,7 +449,7 @@ async def _get_version_location(self): @ensure_connection async def revisions(self, path): """Get past versions of the request file from special user designated version container, - if the user hasn't designated a version_location container in raises an infomative error + if the user hasn't designated a version_location container it raises an infomative error message. The revision endpoint also doesn't return the current version so that is added to the revision list after other revisions are returned. More info about versioning with Cloud Files here: @@ -475,7 +474,7 @@ async def revisions(self, path): return [current] + [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] @ensure_connection - async def get_metadata_revision(self, path, version=None, revision=None): + async def _metadata_revision(self, path, version=None, revision=None): version_location = await self._get_version_location() resp = await self.make_request( From 3ad9fbe7c082a709143a50767022dfd2a7143b6e Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 16 Nov 2017 09:51:45 -0500 Subject: [PATCH 09/15] remove blank lines --- tests/providers/cloudfiles/fixtures.py | 8 +++++--- tests/providers/cloudfiles/test_provider.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py index deb6148f3..0e26ddbbd 100644 --- a/tests/providers/cloudfiles/fixtures.py +++ b/tests/providers/cloudfiles/fixtures.py @@ -12,7 +12,6 @@ from waterbutler.providers.cloudfiles import CloudFilesProvider - @pytest.fixture def auth(): return { @@ -105,11 +104,11 @@ def file_stream(file_like): return streams.FileStreamReader(file_like) - @pytest.fixture def folder_root_empty(): return [] + @pytest.fixture def container_header_metadata_with_verision_location(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: @@ -121,11 +120,13 @@ def container_header_metadata_without_verision_location(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: return json.load(fp)['container_header_metadata_without_verision_location'] + @pytest.fixture def file_metadata(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: return json.load(fp)['file_metadata'] + @pytest.fixture def folder_root(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: @@ -161,11 +162,13 @@ def folder_root(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: return json.load(fp)['folder_root'] + @pytest.fixture def revision_list(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: return json.load(fp)['revision_list'] + @pytest.fixture def file_root_level1_level2_file2_txt(): return aiohttp.multidict.CIMultiDict([ @@ -226,7 +229,6 @@ def file_root_similar_name(): ]) - @pytest.fixture def file_header_metadata_txt(): return aiohttp.multidict.CIMultiDict([ diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index 58c66d71a..40e6c44b9 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -80,7 +80,6 @@ async def test_download_revision(self, revision_url = connected_provider.build_url(version_name, container='versions-container', ) aiohttpretty.register_uri('GET', revision_url, body=body, auto_length=True) - result = await connected_provider.download(path, version=version_name) content = await result.read() From 3e62ede5ddc6125770dd406d24cbfcf44e88e771 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 16 Nov 2017 10:25:27 -0500 Subject: [PATCH 10/15] remove blank lines --- tests/providers/cloudfiles/fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py index 0e26ddbbd..de8788b6c 100644 --- a/tests/providers/cloudfiles/fixtures.py +++ b/tests/providers/cloudfiles/fixtures.py @@ -39,7 +39,6 @@ def provider(auth, credentials, settings): return CloudFilesProvider(auth, credentials, settings) - @pytest.fixture def token(auth_json): return auth_json['access']['token']['id'] From e4ad0a95c00a3210c4090b2a0645122e50034c0a Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 17 Nov 2017 10:12:08 -0500 Subject: [PATCH 11/15] Standardize import indentation --- tests/providers/cloudfiles/test_metadata.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index e304b3dce..5904116e3 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -1,10 +1,12 @@ import pytest from waterbutler.core.path import WaterButlerPath -from waterbutler.providers.cloudfiles.metadata import (CloudFilesFileMetadata, - CloudFilesHeaderMetadata, - CloudFilesFolderMetadata, - CloudFilesRevisonMetadata) +from waterbutler.providers.cloudfiles.metadata import ( + CloudFilesFileMetadata, + CloudFilesHeaderMetadata, + CloudFilesFolderMetadata, + CloudFilesRevisonMetadata +) from tests.providers.cloudfiles.fixtures import ( file_header_metadata_txt, From 64c9a23b9e6776099c7716b595bc756cb5fbce58 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 27 Nov 2017 11:28:56 -0500 Subject: [PATCH 12/15] reorganize delete methods and add tests for deleting the root to Cloudfiles provider. --- tests/providers/cloudfiles/test_provider.py | 26 ++++++++++ waterbutler/providers/cloudfiles/provider.py | 53 ++++++++++++-------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index 40e6c44b9..8b0e500ca 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -218,6 +218,32 @@ async def test_delete_folder(self, connected_provider, folder_root_empty, file_h assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) assert aiohttpretty.has_call(method='DELETE', uri=delete_url_folder) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_root(self, connected_provider, folder_root_empty, file_header_metadata): + + path = WaterButlerPath('/') + query = {'prefix': path.path} + url = connected_provider.build_url('', **query) + body = json.dumps(folder_root_empty).encode('utf-8') + + delete_query = {'bulk-delete': ''} + delete_url_folder = connected_provider.build_url(path.name, **delete_query) + delete_url_content = connected_provider.build_url('', **delete_query) + + file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('GET', url, body=body) + aiohttpretty.register_uri('HEAD', file_url, headers=file_header_metadata) + + aiohttpretty.register_uri('DELETE', delete_url_content, status=200) + + with pytest.raises(exceptions.DeleteError): + await connected_provider.delete(path) + + await connected_provider.delete(path, confirm_delete=1) + + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_delete_file(self, connected_provider): diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 68c1da020..afb5836ee 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -155,7 +155,7 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): return metadata, created @ensure_connection - async def delete(self, path, confirm_delete=False): + async def delete(self, path, confirm_delete=0): """Deletes the key at the specified path :param str path: The path of the key to delete :param int confirm_delete: Must be 1 to confirm root folder delete, this deletes entire @@ -170,26 +170,37 @@ async def delete(self, path, confirm_delete=False): ) if path.is_dir: - metadata = await self._metadata_folder(path, recursive=True) - - delete_files = [ - os.path.join('/', self.container, path.child(item.name).path) - for item in metadata - ] - - delete_files.append(os.path.join('/', self.container, path.path)) - query = {'bulk-delete': ''} - resp = await self.make_request( - 'DELETE', - functools.partial(self.build_url, '', **query), - data='\n'.join(delete_files), - expects=(200, ), - throws=exceptions.DeleteError, - headers={ - 'Content-Type': 'text/plain', - }, - ) - await resp.release() + await self._delete_folder_contents(path) + + if not path.is_root: # deleting the root "item" deletes the whole bucket. + await self._delete_item(path) + + @ensure_connection + async def _delete_folder_contents(self, path): + + metadata = await self._metadata_folder(path, recursive=True) + + delete_files = [ + os.path.join('/', self.container, path.child(item.name).path) + for item in metadata + ] + + delete_files.append(os.path.join('/', self.container, path.path)) + query = {'bulk-delete': ''} + resp = await self.make_request( + 'DELETE', + functools.partial(self.build_url, '', **query), + data='\n'.join(delete_files), + expects=(200,), + throws=exceptions.DeleteError, + headers={ + 'Content-Type': 'text/plain', + }, + ) + await resp.release() + + @ensure_connection + async def _delete_item(self, path): resp = await self.make_request( 'DELETE', From 9a58bed932daa6e3c7b6214d50ea674ac6717d37 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 4 Dec 2017 14:49:18 -0500 Subject: [PATCH 13/15] add kwargs for registrations --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index afb5836ee..e0c426251 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -118,7 +118,7 @@ async def download(self, return streams.ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True): + async def upload(self, stream, path, check_created=True, fetch_metadata=True, **kwargs): """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into From 78230fca548b2030cdaaeaa6b567756d730fa35f Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Wed, 10 Jan 2018 13:36:28 -0500 Subject: [PATCH 14/15] style changes and add type hinting to Cloudfiles --- tests/providers/cloudfiles/test_provider.py | 41 +++-- waterbutler/providers/cloudfiles/metadata.py | 4 +- waterbutler/providers/cloudfiles/provider.py | 167 +++++++++---------- 3 files changed, 108 insertions(+), 104 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index 8b0e500ca..a7015d274 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -124,14 +124,14 @@ async def test_upload(self, metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') aiohttpretty.register_uri('HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_header_metadata} - ] + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_header_metadata} + ] ) - aiohttpretty.register_uri('PUT', url, status=200, + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"{}"'.format(content_md5)}) metadata, created = await connected_provider.upload(file_stream, path) @@ -147,7 +147,7 @@ async def test_create_folder(self, connected_provider, file_header_metadata): path = WaterButlerPath('/foo/', folder=True) metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri('PUT', url, status=200) + aiohttpretty.register_uri('PUT', url, status=201) aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) metadata = await connected_provider.create_folder(path) @@ -165,7 +165,7 @@ async def test_upload_check_none(self, connected_provider, metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) - aiohttpretty.register_uri('PUT', url, status=200, + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"{}"'.format(content_md5)}) metadata, created = await connected_provider.upload( file_stream, path, check_created=False, fetch_metadata=False) @@ -185,7 +185,7 @@ async def test_upload_checksum_mismatch(self, url = connected_provider.sign_url(path, 'PUT') aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) - aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"Bogus MD5"'}) + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"Bogus MD5"'}) with pytest.raises(exceptions.UploadChecksumMismatchError): await connected_provider.upload(file_stream, path) @@ -374,9 +374,11 @@ async def test_metadata_404(self, connected_provider): aiohttpretty.register_uri('GET', url, status=200, body=b'') connected_provider._metadata_item = MockCoroutine(return_value=None) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) + assert exc.value.message == "Could not retrieve folder '/level1/'" + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_root_level1(self, connected_provider, folder_root_level1): @@ -469,9 +471,11 @@ async def test_metadata_file_does_not_exist(self, connected_provider): path = WaterButlerPath('/does_not.exist') url = connected_provider.build_url(path.path) aiohttpretty.register_uri('HEAD', url, status=404) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) + assert exc.value.message == "Could not retrieve '/does_not.exist'" + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_does_not_exist(self, connected_provider): @@ -481,9 +485,11 @@ async def test_metadata_folder_does_not_exist(self, connected_provider): file_url = connected_provider.build_url(path.path.rstrip('/')) aiohttpretty.register_uri('GET', folder_url, status=200, body=folder_body) aiohttpretty.register_uri('HEAD', file_url, status=404) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) + assert exc.value.message == "Could not retrieve '/does_not_exist/'" + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_file_bad_content_type(self, connected_provider, file_metadata): @@ -492,9 +498,10 @@ async def test_metadata_file_bad_content_type(self, connected_provider, file_met path = WaterButlerPath('/does_not.exist') url = connected_provider.build_url(path.path) aiohttpretty.register_uri('HEAD', url, headers=item) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) + assert exc.value.message == "Could not retrieve '/does_not.exist'" class TestV1ValidatePath: @@ -532,9 +539,15 @@ async def test_no_version_location(self, container_url, headers=container_header_metadata_without_verision_location) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.revisions(path) + assert exc.value.message == 'The your container does not have a defined version location.' \ + ' To set a version location and store file versions follow' \ + ' the instructions here: https://developer.rackspace.com/' \ + 'docs/cloud-files/v1/use-cases/' \ + 'additional-object-services-information/#object-versioning' + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_ensure_connection_not_public(self, provider, auth_json, temp_url_key): diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index 5e44720b1..90b124304 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -93,13 +93,13 @@ def content_type(self): @property def etag(self): - return self.raw['ETAG'] + return self.raw['ETAG'].replace('"', '') @property def extra(self): return { 'hashes': { - 'md5': self.raw['ETAG'], + 'md5': self.raw['ETAG'].replace('"', '') }, } diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index e0c426251..6f2ceb00d 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -2,6 +2,8 @@ import hmac import json import time +import typing +from typing import List, Union, Tuple import asyncio import hashlib import functools @@ -9,7 +11,7 @@ import furl -from waterbutler.core import streams +from waterbutler.core.streams import ResponseStreamReader, HashStreamWriter from waterbutler.core import provider from waterbutler.core import exceptions from waterbutler.core.path import WaterButlerPath @@ -29,7 +31,7 @@ def ensure_connection(func): @functools.wraps(func) async def wrapped(self, *args, **kwargs): await self._ensure_connection() - return (await func(self, *args, **kwargs)) + return await func(self, *args, **kwargs) return wrapped @@ -53,10 +55,10 @@ def __init__(self, auth, credentials, settings): self.use_public = self.settings.get('use_public', True) self.metrics.add('region', self.region) - async def validate_v1_path(self, path, **kwargs): + async def validate_v1_path(self, path: str, **kwargs): return await self.validate_path(path, **kwargs) - async def validate_path(self, path, **kwargs): + async def validate_path(self, path: str, **kwargs): return WaterButlerPath(path) @property @@ -76,7 +78,7 @@ async def intra_copy(self, dest_provider, source_path, dest_path): headers={ 'X-Copy-From': os.path.join(self.container, source_path.path) }, - expects=(201, ), + expects=(201,), throws=exceptions.IntraCopyError, ) await resp.release() @@ -84,20 +86,14 @@ async def intra_copy(self, dest_provider, source_path, dest_path): @ensure_connection async def download(self, - path, - accept_url=False, - range=None, - version=None, - revision=None, - displayName=None, - **kwargs): - """Returns a ResponseStreamReader (Stream) for the specified path - :param str path: Path to the object you want to download - :param dict \*\*kwargs: Additional arguments that are ignored - :rtype str: - :rtype ResponseStreamReader: - :raises: exceptions.DownloadError - """ + path: WaterButlerPath, + accept_url: bool=False, + range: tuple=None, + version: str=None, + revision: str=None, + displayName: str=None, + **kwargs) -> ResponseStreamReader: + """Returns a ResponseStreamReader (Stream) for the specified path """ self.metrics.add('download.accept_url', accept_url) if accept_url: parsed_url = furl.furl(self.sign_url(path, endpoint=self.public_endpoint)) @@ -115,10 +111,15 @@ async def download(self, expects=(200, 206), throws=exceptions.DownloadError, ) - return streams.ResponseStreamReader(resp) + return ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True, **kwargs): + async def upload(self, + stream: ResponseStreamReader, + path: WaterButlerPath, + check_created: bool=True, + fetch_metadata: bool=True, + **kwargs) -> Tuple[CloudFilesHeaderMetadata, bool]: """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into @@ -132,13 +133,13 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** created = None self.metrics.add('upload.check_created', check_created) - stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5)) + stream.add_writer('md5', HashStreamWriter(hashlib.md5)) resp = await self.make_request( 'PUT', functools.partial(self.sign_url, path, 'PUT'), data=stream, headers={'Content-Length': str(stream.size)}, - expects=(200, 201), + expects=(201,), throws=exceptions.UploadError, ) await resp.release() @@ -155,7 +156,7 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** return metadata, created @ensure_connection - async def delete(self, path, confirm_delete=0): + async def delete(self, path: WaterButlerPath, confirm_delete: int=0) -> None: """Deletes the key at the specified path :param str path: The path of the key to delete :param int confirm_delete: Must be 1 to confirm root folder delete, this deletes entire @@ -176,7 +177,7 @@ async def delete(self, path, confirm_delete=0): await self._delete_item(path) @ensure_connection - async def _delete_folder_contents(self, path): + async def _delete_folder_contents(self, path: WaterButlerPath) -> None: metadata = await self._metadata_folder(path, recursive=True) @@ -200,7 +201,7 @@ async def _delete_folder_contents(self, path): await resp.release() @ensure_connection - async def _delete_item(self, path): + async def _delete_item(self, path: WaterButlerPath) -> None: resp = await self.make_request( 'DELETE', @@ -211,20 +212,25 @@ async def _delete_item(self, path): await resp.release() @ensure_connection - async def metadata(self, path, recursive=False, version=None, revision=None, **kwargs): - """Get Metadata about the requested file or folder - :param str path: The path to a key or folder - :rtype dict: - :rtype list: - """ + async def metadata(self, path: WaterButlerPath, + recursive: bool=False, + version: str=None, + revision: str=None, + **kwargs) -> Union[CloudFilesHeaderMetadata, List]: + """Get Metadata about the requested file or the metadata of a folder's contents""" + if path.is_dir: - return (await self._metadata_folder(path, recursive=recursive)) + return await self._metadata_folder(path, recursive=recursive) elif version or revision: - return (await self._metadata_revision(path, version, revision)) + return await self._metadata_revision(path, version, revision) else: - return (await self._metadata_item(path)) + return await self._metadata_item(path) - def build_url(self, path, _endpoint=None, container=None, **query): + def build_url(self, + path: str, + _endpoint: str=None, + container: str=None, + **query) -> str: """Build the url for the specified object :param args segments: URI segments :param kwargs query: Query parameters @@ -243,14 +249,12 @@ def can_intra_copy(self, dest_provider, path=None): def can_intra_move(self, dest_provider, path=None): return type(self) == type(dest_provider) and not getattr(path, 'is_dir', False) - def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_SECS): - """Sign a temp url for the specified stream - :param str stream: The requested stream's path - :param CloudFilesPath path: A path to a file/folder - :param str method: The HTTP method used to access the returned url - :param int seconds: Time for the url to live - :rtype str: - """ + def sign_url(self, + path: WaterButlerPath, + method: str='GET', + endpoint: str=None, + seconds: int=settings.TEMP_URL_SECS) -> str: + """Sign a temp url for the specified stream""" method = method.upper() expires = str(int(time.time() + seconds)) @@ -258,8 +262,8 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ body = '\n'.join([method, expires]) body += '\n' + str(url.path) - body = unquote(body).encode() - signature = hmac.new(self.temp_url_key, body, hashlib.sha1).hexdigest() + body_data = unquote(body).encode() + signature = hmac.new(self.temp_url_key, body_data, hashlib.sha1).hexdigest() url.args.update({ 'temp_url_sig': signature, 'temp_url_expires': expires, @@ -269,12 +273,12 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ async def make_request(self, *args, **kwargs): try: - return (await super().make_request(*args, **kwargs)) + return await super().make_request(*args, **kwargs) except exceptions.ProviderError as e: if e.code != 408: raise await asyncio.sleep(1) - return (await super().make_request(*args, **kwargs)) + return await super().make_request(*args, **kwargs) async def _ensure_connection(self): """Defines token, endpoint and temp_url_key if they are not already defined @@ -302,24 +306,19 @@ async def _ensure_connection(self): except KeyError: raise exceptions.ProviderError('No temp url key is available', code=503) - def _extract_endpoints(self, data): - """Pulls both the public and internal cloudfiles urls, - returned respectively, from the return of tokens - Very optimized. - :param dict data: The json response from the token endpoint - :rtype (str, str): - """ + def _extract_endpoints(self, data: dict): + """Pulls both the public and internal cloudfiles urls, returned respectively, from the + return of tokens Very optimized.""" for service in reversed(data['access']['serviceCatalog']): if service['name'].lower() == 'cloudfiles': for region in service['endpoints']: if region['region'].lower() == self.region.lower(): return region['publicURL'], region['internalURL'] - async def _get_token(self): + async def _get_token(self) -> dict: """Fetches an access token from cloudfiles for actual api requests Returns the entire json response from the tokens endpoint Notably containing our token and proper endpoint to send requests to - :rtype dict: """ resp = await self.make_request( 'POST', @@ -337,37 +336,29 @@ async def _get_token(self): }, expects=(200, ), ) - data = await resp.json() - return data + return await resp.json() - async def _metadata_item(self, path): - """Get Metadata about the requested file - :param str path: The path to a key - :rtype dict: - :rtype list: - """ + async def _metadata_item(self, path: WaterButlerPath) -> CloudFilesHeaderMetadata: + """Get Metadata about the requested file or folder""" resp = await self.make_request( 'HEAD', functools.partial(self.build_url, path.path), - expects=(200, ), + expects=(200, 404), throws=exceptions.MetadataError, ) await resp.release() - if resp.headers['Content-Type'] == 'application/directory' and path.is_file: + if resp.status == 404 or resp.headers['Content-Type'] == 'application/directory' and path.is_file: raise exceptions.MetadataError( - 'Could not retrieve file \'{0}\''.format(str(path)), + 'Could not retrieve \'{0}\''.format(str(path)), code=404, ) return CloudFilesHeaderMetadata(dict(resp.headers), path) - async def _metadata_folder(self, path, recursive=False): - """Get Metadata about the requested folder - :param str path: The path to a folder - :rtype dict: - :rtype list: - """ + async def _metadata_folder(self, path: WaterButlerPath, recursive: bool=False) -> \ + List[Union[CloudFilesFolderMetadata, CloudFilesFileMetadata]]: + """Get Metadata about the contents of requested folder""" # prefix must be blank when searching the root of the container query = {'prefix': path.path} self.metrics.add('metadata.folder.is_recursive', True if recursive else False) @@ -384,7 +375,7 @@ async def _metadata_folder(self, path, recursive=False): # no data and the provider path is not root, we are left with either a file or a directory # marker if not data and not path.is_root: - # Convert the parent path into a directory marker (file) and check for an empty folder + # Convert the parent path into a directory marker (item) and check for an empty folder dir_marker = path.parent.child(path.name, folder=path.is_dir) metadata = await self._metadata_item(dir_marker) if not metadata: @@ -408,7 +399,8 @@ async def _metadata_folder(self, path, recursive=False): for item in data ] - def _serialize_metadata(self, data): + def _serialize_metadata(self, data: dict) -> typing.Union[CloudFilesFolderMetadata, + CloudFilesFileMetadata]: if data.get('subdir'): return CloudFilesFolderMetadata(data) elif data['content_type'] == 'application/directory': @@ -416,15 +408,11 @@ def _serialize_metadata(self, data): return CloudFilesFileMetadata(data) @ensure_connection - async def create_folder(self, path, **kwargs): + async def create_folder(self, path: WaterButlerPath, **kwargs) -> CloudFilesHeaderMetadata: """Create a folder in the current provider at `path`. Returns a `BaseFolderMetadata` object if successful. May throw a 409 Conflict if a directory with the same name already exists. Enpoint information can be found here: https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/pseudo-hierarchical-folders-and-directories/ - :param path: ( :class:`.WaterButlerPath` )User-supplied path to create. Must be a directory. - :param kwargs: dict unused params - :rtype: :class:`.BaseFileMetadata` - :raises: :class:`.CreateFolderError` """ resp = await self.make_request( @@ -438,7 +426,7 @@ async def create_folder(self, path, **kwargs): return await self._metadata_item(path) @ensure_connection - async def _get_version_location(self): + async def _get_version_location(self) -> str: resp = await self.make_request( 'HEAD', functools.partial(self.build_url, ''), @@ -458,8 +446,8 @@ async def _get_version_location(self): '#object-versioning') @ensure_connection - async def revisions(self, path): - """Get past versions of the request file from special user designated version container, + async def revisions(self, path: WaterButlerPath, **kwarg) -> List[CloudFilesRevisonMetadata]: + """Get past versions of the requested file from special user designated version container, if the user hasn't designated a version_location container it raises an infomative error message. The revision endpoint also doesn't return the current version so that is added to the revision list after other revisions are returned. More info about versioning with Cloud @@ -485,7 +473,10 @@ async def revisions(self, path): return [current] + [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] @ensure_connection - async def _metadata_revision(self, path, version=None, revision=None): + async def _metadata_revision(self, + path: WaterButlerPath, + version: str=None, + revision: str=None) -> CloudFilesHeaderMetadata: version_location = await self._get_version_location() resp = await self.make_request( @@ -498,16 +489,16 @@ async def _metadata_revision(self, path, version=None, revision=None): return CloudFilesHeaderMetadata(resp.headers, path) - async def _download_revision(self, range, version): + async def _download_revision(self, request_range: tuple, version: str) -> ResponseStreamReader: version_location = await self._get_version_location() resp = await self.make_request( 'GET', functools.partial(self.build_url, version, container=version_location), - range=range, + range=request_range, expects=(200, 206), throws=exceptions.DownloadError ) - return streams.ResponseStreamReader(resp) + return ResponseStreamReader(resp) From e63f8bfb29e0933825080cd59c31e5223e42049f Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 11 Jan 2018 14:16:40 -0500 Subject: [PATCH 15/15] fix recursive delete and clean up docstrings. --- tests/providers/cloudfiles/test_provider.py | 29 +++------ waterbutler/providers/cloudfiles/provider.py | 64 +++++++++----------- 2 files changed, 37 insertions(+), 56 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index a7015d274..6e688282c 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -198,7 +198,7 @@ async def test_upload_checksum_mismatch(self, async def test_delete_folder(self, connected_provider, folder_root_empty, file_header_metadata): path = WaterButlerPath('/delete/') - query = {'prefix': path.path} + query = {'prefix': path.path, 'delimiter': '/'} url = connected_provider.build_url('', **query) body = json.dumps(folder_root_empty).encode('utf-8') @@ -215,7 +215,6 @@ async def test_delete_folder(self, connected_provider, folder_root_empty, file_h await connected_provider.delete(path) - assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) assert aiohttpretty.has_call(method='DELETE', uri=delete_url_folder) @pytest.mark.asyncio @@ -223,7 +222,7 @@ async def test_delete_folder(self, connected_provider, folder_root_empty, file_h async def test_delete_root(self, connected_provider, folder_root_empty, file_header_metadata): path = WaterButlerPath('/') - query = {'prefix': path.path} + query = {'prefix': path.path, 'delimiter': '/'} url = connected_provider.build_url('', **query) body = json.dumps(folder_root_empty).encode('utf-8') @@ -237,12 +236,14 @@ async def test_delete_root(self, connected_provider, folder_root_empty, file_hea aiohttpretty.register_uri('DELETE', delete_url_content, status=200) - with pytest.raises(exceptions.DeleteError): + with pytest.raises(exceptions.DeleteError) as exc: await connected_provider.delete(path) + assert exc.value.message == 'query argument confirm_delete=1 is required for' \ + ' deleting the entire root contents.' + await connected_provider.delete(path, confirm_delete=1) - assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -377,7 +378,7 @@ async def test_metadata_404(self, connected_provider): with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) - assert exc.value.message == "Could not retrieve folder '/level1/'" + assert exc.value.message == "'/level1/' could not be found." @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -474,7 +475,7 @@ async def test_metadata_file_does_not_exist(self, connected_provider): with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) - assert exc.value.message == "Could not retrieve '/does_not.exist'" + assert exc.value.message == "'/does_not.exist' could not be found." @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -488,20 +489,8 @@ async def test_metadata_folder_does_not_exist(self, connected_provider): with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) - assert exc.value.message == "Could not retrieve '/does_not_exist/'" - - @pytest.mark.asyncio - @pytest.mark.aiohttpretty - async def test_metadata_file_bad_content_type(self, connected_provider, file_metadata): - item = file_metadata - item['Content-Type'] = 'application/directory' - path = WaterButlerPath('/does_not.exist') - url = connected_provider.build_url(path.path) - aiohttpretty.register_uri('HEAD', url, headers=item) - with pytest.raises(exceptions.MetadataError) as exc: - await connected_provider.metadata(path) + assert exc.value.message == "'/does_not_exist/' could not be found." - assert exc.value.message == "Could not retrieve '/does_not.exist'" class TestV1ValidatePath: diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 6f2ceb00d..912d17719 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -88,7 +88,7 @@ async def intra_copy(self, dest_provider, source_path, dest_path): async def download(self, path: WaterButlerPath, accept_url: bool=False, - range: tuple=None, + request_range: tuple=None, version: str=None, revision: str=None, displayName: str=None, @@ -102,12 +102,12 @@ async def download(self, version = revision or version if version: - return await self._download_revision(range, version) + return await self._download_revision(request_range, version) resp = await self.make_request( 'GET', functools.partial(self.sign_url, path), - range=range, + range=request_range, expects=(200, 206), throws=exceptions.DownloadError, ) @@ -157,36 +157,33 @@ async def upload(self, @ensure_connection async def delete(self, path: WaterButlerPath, confirm_delete: int=0) -> None: - """Deletes the key at the specified path - :param str path: The path of the key to delete - :param int confirm_delete: Must be 1 to confirm root folder delete, this deletes entire - container object. - :rtype None: - """ + """Deletes the key at the specified path.""" - if path.is_root and not confirm_delete: + if path.is_root and confirm_delete != 1: raise exceptions.DeleteError( - 'query argument confirm_delete=1 is required for deleting the entire container.', + 'query argument confirm_delete=1 is required for' + ' deleting the entire root contents.', code=400 ) if path.is_dir: - await self._delete_folder_contents(path) - - if not path.is_root: # deleting the root "item" deletes the whole bucket. + await self._delete_folder(path) + else: await self._delete_item(path) @ensure_connection - async def _delete_folder_contents(self, path: WaterButlerPath) -> None: + async def _delete_folder(self, path: WaterButlerPath) -> None: + """Folders must be emptied of all contents before they can be deleted""" - metadata = await self._metadata_folder(path, recursive=True) + metadata = await self._metadata_folder(path) - delete_files = [ - os.path.join('/', self.container, path.child(item.name).path) - for item in metadata - ] + delete_files = [] + for item in metadata: + if item.kind == 'folder': + await self._delete_folder(path.from_metadata(item)) + else: + delete_files.append(os.path.join('/', self.container, path.child(item.name).path)) - delete_files.append(os.path.join('/', self.container, path.path)) query = {'bulk-delete': ''} resp = await self.make_request( 'DELETE', @@ -200,6 +197,9 @@ async def _delete_folder_contents(self, path: WaterButlerPath) -> None: ) await resp.release() + if not path.is_root: # deleting root here would destory the container + await self._delete_item(path) + @ensure_connection async def _delete_item(self, path: WaterButlerPath) -> None: @@ -213,14 +213,13 @@ async def _delete_item(self, path: WaterButlerPath) -> None: @ensure_connection async def metadata(self, path: WaterButlerPath, - recursive: bool=False, version: str=None, revision: str=None, **kwargs) -> Union[CloudFilesHeaderMetadata, List]: """Get Metadata about the requested file or the metadata of a folder's contents""" if path.is_dir: - return await self._metadata_folder(path, recursive=recursive) + return await self._metadata_folder(path) elif version or revision: return await self._metadata_revision(path, version, revision) else: @@ -231,11 +230,7 @@ def build_url(self, _endpoint: str=None, container: str=None, **query) -> str: - """Build the url for the specified object - :param args segments: URI segments - :param kwargs query: Query parameters - :rtype str: - """ + """Build the url for the specified object.""" endpoint = _endpoint or self.endpoint container = container or self.container return provider.build_url(endpoint, container, *path.split('/'), **query) @@ -348,22 +343,19 @@ async def _metadata_item(self, path: WaterButlerPath) -> CloudFilesHeaderMetadat ) await resp.release() - if resp.status == 404 or resp.headers['Content-Type'] == 'application/directory' and path.is_file: + if resp.status == 404: raise exceptions.MetadataError( - 'Could not retrieve \'{0}\''.format(str(path)), + '\'{}\' could not be found.'.format(str(path)), code=404, ) return CloudFilesHeaderMetadata(dict(resp.headers), path) - async def _metadata_folder(self, path: WaterButlerPath, recursive: bool=False) -> \ + async def _metadata_folder(self, path: WaterButlerPath) -> \ List[Union[CloudFilesFolderMetadata, CloudFilesFileMetadata]]: """Get Metadata about the contents of requested folder""" # prefix must be blank when searching the root of the container - query = {'prefix': path.path} - self.metrics.add('metadata.folder.is_recursive', True if recursive else False) - if not recursive: - query.update({'delimiter': '/'}) + query = {'prefix': path.path, 'delimiter': '/'} resp = await self.make_request( 'GET', functools.partial(self.build_url, '', **query), @@ -380,7 +372,7 @@ async def _metadata_folder(self, path: WaterButlerPath, recursive: bool=False) - metadata = await self._metadata_item(dir_marker) if not metadata: raise exceptions.MetadataError( - 'Could not retrieve folder \'{0}\''.format(str(path)), + '\'{0}\' could not be found.'.format(str(path)), code=404, )