diff --git a/README.md b/README.md index 4301a60..40bd6b5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +Version 3.24.6 + +New Features: + +1. Supported check data integrity by crc64, including ObsClient.putFile,ObsClient.putContent,ObsClient.appendObject,ObsClient.uploadPart,ObsClient.completeMultipartUpload. + +------------------------------------------------------------------------------------------------- Version 3.24.3 New Features: diff --git a/README_CN.md b/README_CN.md index b2cd284..d7bb5bc 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,4 +1,11 @@ -Version 3.24.3 +Version 3.24.6 + +新特性： + +1. 支持crc64校验(ObsClient.putFile/ObsClient.putContent/ObsClient.appendObject/ObsClient.uploadPart/ObsClient.completeMultipartUpload) + +------------------------------------------------------------------------------------------------- +Version 3.24.3 新特性： diff --git a/src/obs/client.py b/src/obs/client.py index 4242879..6e4edd5 100644 --- a/src/obs/client.py +++ b/src/obs/client.py @@ -1692,7 +1692,7 @@ def appendObject(self, bucketName, objectKey, content=None, metadata=None, heade headers, readable, notifier, entity = self._prepare_file_notifier_and_entity(offset, file_size, headers, progressCallback, file_path, readable) - headers = self.convertor.trans_put_object(metadata=metadata, headers=headers) + headers = self.convertor.trans_put_object(metadata=metadata, headers=headers, file_path=file_path) self.log_client.log(DEBUG, 'send Path:%s' % file_path) else: entity = content.get('content') @@ -1701,7 +1701,7 @@ def appendObject(self, bucketName, objectKey, content=None, metadata=None, heade autoClose, readable, chunkedMode, notifier) - headers = self.convertor.trans_put_object(metadata=metadata, headers=headers) + headers = self.convertor.trans_put_object(metadata=metadata, headers=headers, content=content.get('content')) try: if notifier is not None: @@ -1727,7 +1727,7 @@ def putContent(self, bucketName, objectKey, content=None, metadata=None, headers headers = PutObjectHeader() if headers.get('contentType') is None: headers['contentType'] = const.MIME_TYPES.get(objectKey[objectKey.rfind('.') + 1:].lower()) - _headers = self.convertor.trans_put_object(metadata=metadata, headers=headers) + _headers = self.convertor.trans_put_object(metadata=metadata, headers=headers, content=content) readable = False chunkedMode = False @@ -1789,12 +1789,12 @@ def putFile(self, bucketName, objectKey, file_path, metadata=None, headers=None, __file_path = os.path.join(file_path, f) if not const.IS_PYTHON2: if not objectKey: - key = util.safe_trans_to_gb2312('{0}/'.format(os.path.split(file_path)[1]) + f) + key = util.safe_trans_to_gb2312('{0}'.format(os.path.split(file_path)[1]) + f) else: key = '{0}/'.format(objectKey) + util.safe_trans_to_gb2312(f) else: if not objectKey: - key = util.safe_trans_to_gb2312('{0}/'.format(os.path.split(file_path)[1]) + f).decode( + key = util.safe_trans_to_gb2312('{0}'.format(os.path.split(file_path)[1]) + f).decode( 'GB2312').encode('UTF-8') else: key = '{0}/'.format(objectKey) + util.safe_trans_to_gb2312(f).decode('GB2312').encode('UTF-8') @@ -1810,7 +1810,7 @@ def putFile(self, bucketName, objectKey, file_path, metadata=None, headers=None, headers = self._putFileHandleHeader(headers, size, objectKey, file_path) - _headers = self.convertor.trans_put_object(metadata=metadata, headers=headers) + _headers = self.convertor.trans_put_object(metadata=metadata, headers=headers, file_path=file_path) if const.CONTENT_LENGTH_HEADER not in _headers: _headers[const.CONTENT_LENGTH_HEADER] = util.to_string(size) self.log_client.log(DEBUG, 'send Path:%s' % file_path) @@ -1857,13 +1857,16 @@ def _get_part_size(partSize, file_size, offset): partSize = partSize if partSize is not None and 0 < partSize <= (file_size - offset) else file_size - offset return partSize - def _prepare_headers(self, md5, isAttachMd5, file_path, partSize, offset, sseHeader, headers): + def _prepare_headers(self, md5, isAttachMd5, crc64, isAttachCrc64, file_path, partSize, offset, sseHeader, headers): if md5: headers[const.CONTENT_MD5_HEADER] = md5 elif isAttachMd5: headers[const.CONTENT_MD5_HEADER] = util.base64_encode( util.md5_file_encode_by_size_offset(file_path, partSize, offset, self.chunk_size)) - + if crc64: + self.convertor._put_key_value(headers, self.ha.crc64_header(), crc64) + elif isAttachCrc64: + self.convertor._put_key_value(headers, self.ha.crc64_header(), util.calculate_file_crc64(file_path, offset=offset, totalCount=partSize)) if sseHeader is not None: self.convertor._set_sse_header(sseHeader, headers, True) @@ -1879,12 +1882,15 @@ def _prepare_upload_part_notifier(partSize, progressCallback, readable): return readable, notifier - def _get_headers(self, md5, sseHeader, headers): + def _get_headers(self, md5, crc64, isAttachCrc64, content, sseHeader, headers): if md5: headers[const.CONTENT_MD5_HEADER] = md5 if sseHeader is not None: self.convertor._set_sse_header(sseHeader, headers, True) - + if crc64: + headers[self.ha.crc64_header()] = crc64 + elif isAttachCrc64: + headers[self.ha.crc64_header()] = util.calculate_content_crc64(util.covert_string_to_bytes(content)) return headers @staticmethod @@ -1911,7 +1917,7 @@ def _check_file_part_info(self, file_path, offset, partSize): @funcCache def uploadPart(self, bucketName, objectKey, partNumber, uploadId, object=None, isFile=False, partSize=None, offset=0, sseHeader=None, isAttachMd5=False, md5=None, content=None, progressCallback=None, - autoClose=True, extensionHeaders=None): + autoClose=True, isAttachCrc64=False, crc64=None, extensionHeaders=None): self._assert_not_null(partNumber, 'partNumber is empty') self._assert_not_null(uploadId, 'uploadId is empty') @@ -1926,7 +1932,7 @@ def uploadPart(self, bucketName, objectKey, partNumber, uploadId, object=None, i checked_file_part_info = self._check_file_part_info(content, offset, partSize) headers = {const.CONTENT_LENGTH_HEADER: util.to_string(checked_file_part_info["partSize"])} - headers = self._prepare_headers(md5, isAttachMd5, checked_file_part_info["file_path"], + headers = self._prepare_headers(md5, isAttachMd5, crc64, isAttachCrc64, checked_file_part_info["file_path"], checked_file_part_info["partSize"], checked_file_part_info["offset"], sseHeader, headers) @@ -1938,7 +1944,7 @@ def uploadPart(self, bucketName, objectKey, partNumber, uploadId, object=None, i headers = {} if content is not None and hasattr(content, 'read') and callable(content.read): readable = True - headers = self._get_headers(md5, sseHeader, headers) + headers = self._get_headers(md5, crc64, isAttachCrc64, content, sseHeader, headers) if partSize is None: self.log_client.log(DEBUG, 'missing partSize when uploading a readable stream') @@ -1956,7 +1962,7 @@ def uploadPart(self, bucketName, objectKey, partNumber, uploadId, object=None, i entity = content if entity is None: entity = '' - headers = self._get_headers(md5, sseHeader, headers) + headers = self._get_headers(md5, crc64, isAttachCrc64, content, sseHeader, headers) try: if notifier is not None: @@ -1982,7 +1988,7 @@ def check_file_path(file_path): @funcCache def _uploadPartWithNotifier(self, bucketName, objectKey, partNumber, uploadId, content=None, isFile=False, partSize=None, offset=0, sseHeader=None, isAttachMd5=False, md5=None, notifier=None, - extensionHeaders=None, headers=None): + extensionHeaders=None, headers=None, isAttachCrc64=False, crc64=None): self._assert_not_null(partNumber, 'partNumber is empty') self._assert_not_null(uploadId, 'uploadId is empty') @@ -1994,7 +2000,7 @@ def _uploadPartWithNotifier(self, bucketName, objectKey, partNumber, uploadId, c checked_file_part_info = self._check_file_part_info(content, offset, partSize) headers[const.CONTENT_LENGTH_HEADER] = util.to_string(checked_file_part_info["partSize"]) - headers = self._prepare_headers(md5, isAttachMd5, checked_file_part_info["file_path"], + headers = self._prepare_headers(md5, isAttachMd5, crc64, isAttachCrc64, checked_file_part_info["file_path"], checked_file_part_info["partSize"], checked_file_part_info["offset"], sseHeader, headers) @@ -2005,7 +2011,7 @@ def _uploadPartWithNotifier(self, bucketName, objectKey, partNumber, uploadId, c else: if content is not None and hasattr(content, 'read') and callable(content.read): readable = True - headers = self._get_headers(md5, sseHeader, headers) + headers = self._get_headers(md5, crc64, isAttachCrc64, content, sseHeader, headers) if partSize is None: chunkedMode = True @@ -2018,7 +2024,7 @@ def _uploadPartWithNotifier(self, bucketName, objectKey, partNumber, uploadId, c entity = content if entity is None: entity = '' - headers = self._get_headers(md5, sseHeader, headers) + headers = self._get_headers(md5, crc64, isAttachCrc64, content, sseHeader, headers) ret = self._make_put_request(bucketName, objectKey, pathArgs={'partNumber': partNumber, 'uploadId': uploadId}, headers=headers, entity=entity, chunkedMode=chunkedMode, methodName='uploadPart', @@ -2132,16 +2138,16 @@ def copyPart(self, bucketName, objectKey, partNumber, uploadId, copySource, copy @funcCache def completeMultipartUpload(self, bucketName, objectKey, uploadId, completeMultipartUploadRequest, - extensionHeaders=None, encoding_type=None): + isAttachCrc64=False, extensionHeaders=None, encoding_type=None): self._assert_not_null(uploadId, 'uploadId is empty') self._assert_not_null(completeMultipartUploadRequest, 'completeMultipartUploadRequest is empty') pathArgs = {'uploadId': uploadId} if encoding_type is not None: pathArgs["encoding-type"] = encoding_type + entity, headers = self.convertor.trans_complete_multipart_upload_request(completeMultipartUploadRequest, isAttachCrc64) ret = self._make_post_request(bucketName, objectKey, - pathArgs=pathArgs, - entity=self.convertor.trans_complete_multipart_upload_request( - completeMultipartUploadRequest), methodName='completeMultipartUpload', + pathArgs=pathArgs, headers=headers, + entity=entity, methodName='completeMultipartUpload', extensionHeaders=extensionHeaders) self._generate_object_url(ret, bucketName, objectKey) return ret diff --git a/src/obs/const.py b/src/obs/const.py index 4ea2ef9..2405a97 100644 --- a/src/obs/const.py +++ b/src/obs/const.py @@ -96,7 +96,7 @@ DEFAULT_TASK_NUM = 8 DEFAULT_TASK_QUEUE_SIZE = 20000 -OBS_SDK_VERSION = '3.24.3' +OBS_SDK_VERSION = '3.24.6' V2_META_HEADER_PREFIX = 'x-amz-meta-' V2_HEADER_PREFIX = 'x-amz-' @@ -223,6 +223,7 @@ ALLOWED_RESPONSE_HTTP_HEADER_METADATA_NAMES = ( 'content-type', 'content-md5', + 'checksum-crc64ecma', 'content-length', 'content-language', 'expires', diff --git a/src/obs/convertor.py b/src/obs/convertor.py index 9cb332b..e338cdd 100644 --- a/src/obs/convertor.py +++ b/src/obs/convertor.py @@ -110,6 +110,9 @@ def security_token_header(self): def content_sha256_header(self): return self._get_header_prefix() + 'content-sha256' + def crc64_header(self): + return self._get_header_prefix() + 'checksum-crc64ecma' + def default_storage_class_header(self): return self._get_header_prefix() + 'storage-class' if self.is_obs else 'x-default-storage-class' @@ -680,16 +683,21 @@ def _set_configuration(config_type, urn_type): return ET.tostring(root, 'UTF-8') - @staticmethod - def trans_complete_multipart_upload_request(completeMultipartUploadRequest): + def trans_complete_multipart_upload_request(self, completeMultipartUploadRequest, isAttachCrc64): root = ET.Element('CompleteMultipartUpload') + headers = {} + parts = [] if completeMultipartUploadRequest.get('parts') is None else ( sorted(completeMultipartUploadRequest['parts'], key=lambda d: d.partNum)) + if isAttachCrc64: + object_crc = util.calc_obj_crc_from_parts(parts) + self._put_key_value(headers, self.ha.crc64_header(), object_crc) + for obj in parts: partEle = ET.SubElement(root, 'Part') ET.SubElement(partEle, 'PartNumber').text = util.to_string(obj.get('partNum')) ET.SubElement(partEle, 'ETag').text = util.to_string(obj.get('etag')) - return ET.tostring(root, 'UTF-8') + return (ET.tostring(root, 'UTF-8'), headers) def trans_restore_object(self, **kwargs): pathArgs = {'restore': None} @@ -801,6 +809,8 @@ def trans_restore(self, days, tier): def trans_put_object(self, **kwargs): _headers = {} + file_path = kwargs.get('file_path') + content = kwargs.get('content') metadata = kwargs.get('metadata') headers = kwargs.get('headers') if metadata is not None: @@ -819,7 +829,14 @@ def trans_put_object(self, **kwargs): self.ha.adapt_storage_class(headers.get('storageClass'))) self._put_key_value(_headers, const.CONTENT_LENGTH_HEADER, headers.get('contentLength')) self._put_key_value(_headers, self.ha.expires_header(), headers.get('expires')) - + if headers.get('crc64') is not None: + self._put_key_value(_headers, self.ha.crc64_header(), headers.get('crc64')) + elif headers.get('isAttachCrc64'): + if file_path: + crc64 = util.calculate_file_crc64(file_path) + else: + crc64 = util.calculate_content_crc64(util.covert_string_to_bytes(content)) + self._put_key_value(_headers, self.ha.crc64_header(), crc64) if self.is_obs: self._put_key_value(_headers, self.ha.success_action_redirect_header(), headers.get('successActionRedirect')) @@ -917,7 +934,7 @@ def trans_copy_object(self, **kwargs): self._put_key_value(_headers, self.ha.acl_header(), self.ha.adapt_acl_control(headers.get('acl'))) self._put_key_value(_headers, self.ha.storage_class_header(), self.ha.adapt_storage_class(headers.get('storageClass'))) - + self._put_key_value(_headers, self.ha.crc64_header(), headers.get('crc64')) self._put_key_value(_headers, self.ha.metadata_directive_header(), headers.get('directive')) self._put_key_value(_headers, self.ha.copy_source_if_match_header(), headers.get('if_match')) self._put_key_value(_headers, self.ha.copy_source_if_none_match_header(), headers.get('if_none_match')) @@ -1711,6 +1728,7 @@ def parseCompleteMultipartUpload(self, xml, headers=None): completeMultipartUploadResponse.sseKms = headers.get(self.ha.sse_kms_header()) completeMultipartUploadResponse.sseKmsKey = headers.get(self.ha.sse_kms_key_header()) completeMultipartUploadResponse.sseC = headers.get(self.ha.sse_c_header()) + completeMultipartUploadResponse.crc64 = headers.get(self.ha.crc64_header()) completeMultipartUploadResponse.sseCKeyMd5 = headers.get(self.ha.sse_c_key_md5_header().lower()) return completeMultipartUploadResponse @@ -1841,6 +1859,7 @@ def parsePutContent(self, headers): option.sseC = headers.get(self.ha.sse_c_header()) option.sseCKeyMd5 = headers.get(self.ha.sse_c_key_md5_header().lower()) option.etag = headers.get(const.ETAG_HEADER.lower()) + option.crc64 = headers.get(self.ha.crc64_header()) return option def parseAppendObject(self, headers): @@ -1852,6 +1871,7 @@ def parseAppendObject(self, headers): option.sseCKeyMd5 = headers.get(self.ha.sse_c_key_md5_header().lower()) option.etag = headers.get(const.ETAG_HEADER.lower()) option.nextPosition = util.to_long(headers.get(self.ha.next_position_header())) + option.crc64 = headers.get(self.ha.crc64_header()) return option def parseInitiateMultipartUpload(self, xml, headers=None): @@ -1902,6 +1922,8 @@ def _parseGetObjectCommonHeader(self, headers, option): option.contentLength = util.to_long(headers.get(const.CONTENT_LENGTH_HEADER.lower())) option.contentType = headers.get(const.CONTENT_TYPE_HEADER.lower()) option.lastModified = headers.get(const.LAST_MODIFIED_HEADER.lower()) + option.crc64 = headers.get(self.ha.crc64_header()) + def parseGetObjectMetadata(self, headers): option = GetObjectMetadataResponse() @@ -1940,6 +1962,7 @@ def parseUploadPart(self, headers): uploadPartResponse.sseKmsKey = headers.get(self.ha.sse_kms_key_header()) uploadPartResponse.sseC = headers.get(self.ha.sse_c_header()) uploadPartResponse.sseCKeyMd5 = headers.get(self.ha.sse_c_key_md5_header().lower()) + uploadPartResponse.crc64 = headers.get(self.ha.crc64_header()) return uploadPartResponse def parseCopyPart(self, xml, headers=None): @@ -1952,6 +1975,7 @@ def parseCopyPart(self, xml, headers=None): copyPartResponse.sseKmsKey = headers.get(self.ha.sse_kms_key_header()) copyPartResponse.sseC = headers.get(self.ha.sse_c_header()) copyPartResponse.sseCKeyMd5 = headers.get(self.ha.sse_c_key_md5_header().lower()) + copyPartResponse.crc64 = headers.get(self.ha.crc64_header()) return copyPartResponse def parseGetBucketReplication(self, xml, headers=None): diff --git a/src/obs/crc64mod.py b/src/obs/crc64mod.py new file mode 100644 index 0000000..7bbe27f --- /dev/null +++ b/src/obs/crc64mod.py @@ -0,0 +1,361 @@ +#!/usr/bin/python +# -*- coding:utf-8 -*- +# Copyright 2019 Huawei Technologies Co.,Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +__all__ = '''mkCrcFun Crc +'''.split() + +from obs.const import IS_PYTHON2 +import struct + + +class Crc: + def __init__(self, poly, initCrc=~0, rev=True, xorOut=0, initialize=True): + if not initialize: + return + + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + self.digest_size = sizeBits // 8 + self.poly = poly + self.initCrc = initCrc + self.reverse = rev + self.xorOut = xorOut + + (crcfun, table) = _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut) + self.table = table + self._crc = crcfun + + self.crcValue = self.initCrc + + def __str__(self): + lst = [] + lst.append('reverse = %s' % self.reverse) + lst.append('poly = 0x%X' % self.poly) + fmt = '0x%%0%dX' % (self.digest_size * 2) + lst.append('crcValue = %s' % (fmt % self.crcValue)) + lst.append('xorOut = %s' % (fmt % self.xorOut)) + lst.append('initCrc = %s' % (fmt % self.initCrc)) + return '\n'.join(lst) + + def new(self, arg=None): + n = Crc(poly=None, initialize=False) + n._crc = self._crc + n.table = self.table + n.digest_size = self.digest_size + n.reverse = self.reverse + n.initCrc = self.initCrc + n.crcValue = self.initCrc + n.xorOut = self.xorOut + n.poly = self.poly + if arg is not None: + n.update(arg) + return n + + def copy(self): + c = self.new() + c.crcValue = self.crcValue + return c + + def update(self, data): + self.crcValue = self._crc(data, self.crcValue) + + def digest(self): + n = self.digest_size + crc = self.crcValue + lst = [] + while n > 0: + lst.append(crc & 0xFF) + crc = crc >> 8 + n -= 1 + lst.reverse() + return bytes(lst) + + def hexdigest(self): + crc = self.crcValue + n = self.digest_size + lst = [] + while n > 0: + lst.append('%02X' % (crc & 0xFF)) + crc = crc >> 8 + n -= 1 + lst.reverse() + return ''.join(lst) + + +def mkCrcFun(poly, initCrc=~0, rev=True, xorOut=0): + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + return _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut)[0] + + +def _verifyPoly(poly): + msg = 'The degree of the polynomial must be 64' + n = 64 + low = 1 << n + high = low * 2 + if low <= poly < high: + return n + raise ValueError(msg) + + +def _bitrev(x, n): + y = 0 + for i in range(n): + y = (y << 1) | (x & 1) + x = x >> 1 + return y + + +def _bytecrc(crc, poly, n): + mask = 1 << (n - 1) + for i in range(8): + if crc & mask: + crc = (crc << 1) ^ poly + else: + crc = crc << 1 + mask = (1 << n) - 1 + crc = crc & mask + return crc + + +def _bytecrc_r(crc, poly, n): + for i in range(8): + if crc & 1: + crc = (crc >> 1) ^ poly + else: + crc = crc >> 1 + mask = (1 << n) - 1 + crc = crc & mask + return crc + + +def _mkTable(poly, n): + mask = (1 << n) - 1 + poly = poly & mask + table = [_bytecrc(i << (n - 8), poly, n) for i in range(256)] + return table + + +def _mkTable_r(poly, n): + mask = (1 << n) - 1 + poly = _bitrev(poly & mask, n) + table = [_bytecrc_r(i, poly, n) for i in range(256)] + return table + + +_sizeToTypeCode = {} + +for typeCode in 'B H I L Q'.split(): + size = {1: 8, 2: 16, 4: 32, 8: 64}.get(struct.calcsize(typeCode), None) + if size is not None and size not in _sizeToTypeCode: + _sizeToTypeCode[size] = '256%s' % typeCode + +_sizeToTypeCode[24] = _sizeToTypeCode[32] + +del typeCode, size + + +def _verifyParams(poly, initCrc, xorOut): + sizeBits = _verifyPoly(poly) + mask = (1 << sizeBits) - 1 + initCrc = initCrc & mask + xorOut = xorOut & mask + return (sizeBits, initCrc, xorOut) + + +def _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut): + if rev: + tableList = _mkTable_r(poly, sizeBits) + _fun = _crc64r + else: + tableList = _mkTable(poly, sizeBits) + _fun = _crc64 + + _table = tableList + + if xorOut == 0: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return fun(data, crc, table) + else: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return xorOut ^ fun(data, xorOut ^ crc, table) + + return crcfun, tableList + + +def _get_buffer_view(in_obj): + if isinstance(in_obj, str): + raise TypeError('Unicode-objects must be encoded before calculating a CRC') + mv = memoryview(in_obj) + if mv.ndim > 1: + raise BufferError('Buffer must be single dimension') + return mv + + +def _crc64(data, crc, table): + crc = crc & 0xFFFFFFFFFFFFFFFF + if IS_PYTHON2: + for x in data: + crc = table[ord(x) ^ (int(crc >> 56) & 0xFF)] ^ ((crc << 8) & 0xFFFFFFFFFFFFFF00) + else: + mv = _get_buffer_view(data) + for x in mv.tobytes(): + crc = table[x ^ ((crc >> 56) & 0xFF)] ^ ((crc << 8) & 0xFFFFFFFFFFFFFF00) + return crc + + +def _crc64r(data, crc, table): + crc = crc & 0xFFFFFFFFFFFFFFFF + if IS_PYTHON2: + for x in data: + crc = table[ord(x) ^ int(crc & 0xFF)] ^ (crc >> 8) + else: + mv = _get_buffer_view(data) + for x in mv.tobytes(): + crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8) + return crc + + +import sys + +is_py3 = (sys.version_info[0] == 3) +if is_py3: + xrange = range + long = int + sys.maxint = sys.maxsize + + +def mkCombineFun(poly, initCrc=~long(0), rev=True, xorOut=0): + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + + mask = (long(1) << sizeBits) - 1 + if rev: + poly = _bitrev(long(poly) & mask, sizeBits) + else: + poly = long(poly) & mask + + if sizeBits == 64: + fun = _combine64 + else: + return NotImplemented + + def combine_fun(crc1, crc2, len2): + return fun(poly, initCrc ^ xorOut, rev, xorOut, crc1, crc2, len2) + + return combine_fun + + +GF2_DIM = 64 + + +def gf2_matrix_square(square, mat): + for n in xrange(GF2_DIM): + square[n] = gf2_matrix_times(mat, mat[n]) + + +def gf2_matrix_times(mat, vec): + summary = 0 + mat_index = 0 + + while vec: + if vec & 1: + summary ^= mat[mat_index] + + vec >>= 1 + mat_index += 1 + + return summary + + +def _combine64(poly, initCrc, rev, xorOut, crc1, crc2, len2): + if len2 == 0: + return crc1 + + even = [0] * GF2_DIM + odd = [0] * GF2_DIM + + crc1 ^= initCrc ^ xorOut + + if (rev): + odd[0] = poly + row = 1 + for n in xrange(1, GF2_DIM): + odd[n] = row + row <<= 1 + else: + row = 2 + for n in xrange(0, GF2_DIM - 1): + odd[n] = row + row <<= 1 + odd[GF2_DIM - 1] = poly + + gf2_matrix_square(even, odd) + + gf2_matrix_square(odd, even) + + while True: + gf2_matrix_square(even, odd) + if len2 & long(1): + crc1 = gf2_matrix_times(even, crc1) + len2 >>= 1 + if len2 == 0: + break + + gf2_matrix_square(odd, even) + if len2 & long(1): + crc1 = gf2_matrix_times(odd, crc1) + len2 >>= 1 + + if len2 == 0: + break + + crc1 ^= crc2 + + return crc1 + + +def _verifyPoly(poly): + msg = 'The degree of the polynomial must be 8, 16, 24, 32 or 64' + poly = long(poly) + for n in (8, 16, 24, 32, 64): + low = long(1) << n + high = low * 2 + if low <= poly < high: + return n + raise ValueError(msg) + + +def _bitrev(x, n): + x = long(x) + y = long(0) + for i in xrange(n): + y = (y << 1) | (x & long(1)) + x = x >> 1 + if ((long(1) << n) - 1) <= sys.maxint: + return int(y) + return y + + +def _verifyParams(poly, initCrc, xorOut): + sizeBits = _verifyPoly(poly) + + mask = (long(1) << sizeBits) - 1 + + initCrc = long(initCrc) & mask + if mask <= sys.maxint: + initCrc = int(initCrc) + + xorOut = long(xorOut) & mask + if mask <= sys.maxint: + xorOut = int(xorOut) + + return (sizeBits, initCrc, xorOut) diff --git a/src/obs/model.py b/src/obs/model.py index 7c0f03c..daa8d61 100644 --- a/src/obs/model.py +++ b/src/obs/model.py @@ -184,11 +184,13 @@ def __init__(self, code=None, message=None, status=None, reason=None, body=None, class CompletePart(BaseModel): - allowedAttr = {'partNum': int, 'etag': BASESTRING} + allowedAttr = {'partNum': int, 'etag': BASESTRING, "crc64": [int, LONG, BASESTRING], "size": int} - def __init__(self, partNum=None, etag=None): + def __init__(self, partNum=None, etag=None, crc64=None, size=None): self.partNum = partNum self.etag = etag + self.crc64 = crc64 + self.size = size class AvailableZone(object): @@ -418,11 +420,11 @@ class CopyObjectHeader(BaseModel): 'destSseHeader': SseHeader, 'sourceSseHeader': SseHeader, 'cacheControl': BASESTRING, 'contentDisposition': BASESTRING, 'contentEncoding': BASESTRING, 'contentLanguage': BASESTRING, 'contentType': BASESTRING, - 'expires': BASESTRING, + 'expires': BASESTRING, "crc64": [int, LONG, BASESTRING], 'storageClass': BASESTRING, 'successActionRedirect': BASESTRING, 'extensionGrants': list} def __init__(self, acl=None, directive=None, if_match=None, if_none_match=None, if_modified_since=None, - if_unmodified_since=None, location=None, destSseHeader=None, sourceSseHeader=None, + if_unmodified_since=None, location=None, destSseHeader=None, sourceSseHeader=None, crc64=None, cacheControl=None, contentDisposition=None, contentEncoding=None, contentLanguage=None, contentType=None, expires=None, storageClass=None, successActionRedirect=None, extensionGrants=None): self.acl = acl @@ -443,6 +445,7 @@ def __init__(self, acl=None, directive=None, if_match=None, if_none_match=None, self.storageClass = storageClass self.successActionRedirect = successActionRedirect self.extensionGrants = extensionGrants + self.crc64 = crc64 class SetObjectMetadataHeader(BaseModel): @@ -463,7 +466,8 @@ def __init__(self, removeUnset=False, location=None, cacheControl=None, contentD self.expires = expires self.storageClass = storageClass -class RenameFileHeader(BaseModel,object): + +class RenameFileHeader(BaseModel, object): allowedAttr = {'objectKey': BASESTRING, 'newObjectKey': BASESTRING} def __init__(self, objectKey=None, newObjectKey=None): @@ -492,7 +496,7 @@ class CreateBucketHeader(BaseModel): "isFusionAllowUpgrade": bool, "isFusionAllowAlternative": bool} def __init__(self, aclControl=None, storageClass=None, extensionGrants=None, - availableZone=None, epid=None, isPFS=False, redundancy=None, + availableZone=None, epid=None, isPFS=False, redundancy=None, isFusionAllowUpgrade=None, isFusionAllowAlternative=None): """ Headers that can be carried during bucket creation @@ -516,6 +520,7 @@ def __init__(self, aclControl=None, storageClass=None, extensionGrants=None, self.isFusionAllowUpgrade = isFusionAllowUpgrade self.isFusionAllowAlternative = isFusionAllowAlternative + class ExtensionGrant(BaseModel): allowedAttr = {'permission': BASESTRING, 'granteeId': BASESTRING} @@ -707,10 +712,11 @@ class PutObjectHeader(BaseModel): allowedAttr = {'md5': BASESTRING, 'acl': BASESTRING, 'location': BASESTRING, 'contentType': BASESTRING, 'sseHeader': SseHeader, 'contentLength': [int, LONG, BASESTRING], 'storageClass': BASESTRING, 'successActionRedirect': BASESTRING, 'expires': int, - 'extensionGrants': list, "sha256": BASESTRING} + 'extensionGrants': list, "sha256": BASESTRING, "crc64": [int, LONG, BASESTRING], "isAttachCrc64": bool} def __init__(self, md5=None, acl=None, location=None, contentType=None, sseHeader=None, contentLength=None, - storageClass=None, successActionRedirect=None, expires=None, extensionGrants=None, sha256=None): + storageClass=None, successActionRedirect=None, expires=None, extensionGrants=None, sha256=None, + crc64=None, isAttachCrc64=False): self.md5 = md5 self.sha256 = sha256 self.acl = acl @@ -722,6 +728,8 @@ def __init__(self, md5=None, acl=None, location=None, contentType=None, sseHeade self.successActionRedirect = successActionRedirect self.expires = expires self.extensionGrants = extensionGrants + self.crc64 = crc64 + self.isAttachCrc64 = isAttachCrc64 AppendObjectHeader = PutObjectHeader @@ -949,11 +957,12 @@ def add_part(self, part): class CompleteMultipartUploadResponse(BaseModel): allowedAttr = {'location': BASESTRING, 'bucket': BASESTRING, "encoding_type": BASESTRING, 'key': BASESTRING, 'etag': BASESTRING, 'versionId': BASESTRING, 'sseKms': BASESTRING, - 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, 'objectUrl': BASESTRING} + 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, 'objectUrl': BASESTRING, + 'crc64': BASESTRING} def __init__(self, location=None, bucket=None, key=None, etag=None, versionId=None, sseKms=None, sseKmsKey=None, sseC=None, - sseCKeyMd5=None, objectUrl=None, encoding_type=None): + sseCKeyMd5=None, objectUrl=None, crc64=None, encoding_type=None): self.location = location self.bucket = bucket self.key = key @@ -965,15 +974,16 @@ def __init__(self, location=None, bucket=None, key=None, etag=None, self.sseCKeyMd5 = sseCKeyMd5 self.objectUrl = objectUrl self.encoding_type = encoding_type + self.crc64 = crc64 class CopyObjectResponse(BaseModel): allowedAttr = {'lastModified': BASESTRING, 'etag': BASESTRING, 'copySourceVersionId': BASESTRING, - 'versionId': BASESTRING, + 'versionId': BASESTRING, "crc64": BASESTRING, 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING} def __init__(self, lastModified=None, etag=None, copySourceVersionId=None, versionId=None, sseKms=None, - sseKmsKey=None, sseC=None, sseCKeyMd5=None): + sseKmsKey=None, sseC=None, sseCKeyMd5=None, crc64=None): self.lastModified = lastModified self.etag = etag self.copySourceVersionId = copySourceVersionId @@ -982,14 +992,16 @@ def __init__(self, lastModified=None, etag=None, copySourceVersionId=None, versi self.sseKmsKey = sseKmsKey self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 + self.crc64 = crc64 class CopyPartResponse(BaseModel): allowedAttr = {'lastModified': BASESTRING, 'etag': BASESTRING, 'modifiedDate': BASESTRING, - 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING} + 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, + "crc64": BASESTRING} def __init__(self, lastModified=None, etag=None, modifiedDate=None, sseKms=None, sseKmsKey=None, sseC=None, - sseCKeyMd5=None): + sseCKeyMd5=None, crc64=None): self.lastModified = lastModified self.etag = etag self.modifiedDate = modifiedDate @@ -997,6 +1009,7 @@ def __init__(self, lastModified=None, etag=None, modifiedDate=None, sseKms=None, self.sseKmsKey = sseKmsKey self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 + self.crc64 = crc64 class DeleteObjectResponse(BaseModel): @@ -1088,7 +1101,7 @@ def __init__(self, bucketName=None, objectKey=None, uploadId=None, initiator=Non class GetBucketMetadataResponse(BaseModel): - allowedAttr = {'storageClass': BASESTRING, 'accessContorlAllowOrigin': BASESTRING, + allowedAttr = {'storageClass': BASESTRING, 'accessContorlAllowOrigin': BASESTRING, 'accessContorlAllowHeaders': BASESTRING, 'accessContorlAllowMethods': BASESTRING, 'accessContorlExposeHeaders': BASESTRING, @@ -1119,13 +1132,16 @@ def __init__(self, quota=None): class GetBucketStorageInfoResponse(BaseModel): - allowedAttr = {'size': LONG, 'objectNumber': int, 'standardSize': LONG, 'standardObjectNumber': int, 'warmSize': LONG, - 'warmObjectNumber': int, 'coldSize': LONG, 'coldObjectNumber': int, 'deepArchiveSize': LONG, 'deepArchiveObjectNumber': int, - 'highPerformanceSize': LONG, 'highPerformanceObjectNumber': int, 'standard_IASize': LONG, 'standard_IAObjectNumber': int, 'glacierObjectNumber': LONG} + allowedAttr = {'size': LONG, 'objectNumber': int, 'standardSize': LONG, 'standardObjectNumber': int, + 'warmSize': LONG, 'warmObjectNumber': int, 'coldSize': LONG, 'coldObjectNumber': int, + 'deepArchiveSize': LONG, 'deepArchiveObjectNumber': int, 'highPerformanceSize': LONG, + 'highPerformanceObjectNumber': int, 'standard_IASize': LONG, 'standard_IAObjectNumber': int, + 'glacierObjectNumber': LONG} def __init__(self, size=None, objectNumber=None, standardSize=None, standardObjectNumber=None, warmSize=None, - warmObjectNumber=None, coldSize=None, coldObjectNumber=None, deepArchiveSize=None, deepArchiveObjectNumber=None, - highPerformanceSize=None, highPerformanceObjectNumber=None, standard_IASize=None, standard_IAObjectNumber=None, glacierSize=None, glacierObjectNumber=None): + warmObjectNumber=None, coldSize=None, coldObjectNumber=None, deepArchiveSize=None, + deepArchiveObjectNumber=None, highPerformanceSize=None, highPerformanceObjectNumber=None, + standard_IASize=None, standard_IAObjectNumber=None, glacierSize=None, glacierObjectNumber=None): self.size = size self.objectNumber = objectNumber self.standardSize = standardSize @@ -1143,6 +1159,7 @@ def __init__(self, size=None, objectNumber=None, standardSize=None, standardObje self.glacierSize = glacierSize self.glacierObjectNumber = glacierObjectNumber + class GetBucketEncryptionResponse(BaseModel): allowedAttr = {'encryption': BASESTRING, 'key': BASESTRING} @@ -1166,14 +1183,14 @@ class GetObjectMetadataResponse(BaseModel): 'lastModified': BASESTRING, 'etag': BASESTRING, 'versionId': BASESTRING, 'restore': BASESTRING, 'expiration': BASESTRING, 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, 'isAppendable': bool, - 'nextPosition': LONG} + 'nextPosition': LONG, "crc64": BASESTRING} def __init__(self, storageClass=None, accessContorlAllowOrigin=None, accessContorlAllowHeaders=None, accessContorlAllowMethods=None, accessContorlExposeHeaders=None, accessContorlMaxAge=None, contentLength=None, contentType=None, websiteRedirectLocation=None, lastModified=None, etag=None, versionId=None, restore=None, expiration=None, sseKms=None, sseKmsKey=None, sseC=None, sseCKeyMd5=None, - isAppendable=None, nextPosition=None): + isAppendable=None, nextPosition=None, crc64=None): self.storageClass = storageClass self.accessContorlAllowOrigin = accessContorlAllowOrigin self.accessContorlAllowHeaders = accessContorlAllowHeaders @@ -1195,6 +1212,7 @@ def __init__(self, storageClass=None, accessContorlAllowOrigin=None, accessConto self.sseCKeyMd5 = sseCKeyMd5 self.isAppendable = isAppendable self.nextPosition = nextPosition + self.crc64 = crc64 SetObjectMetadataResponse = GetObjectMetadataResponse @@ -1237,7 +1255,8 @@ def __init__(self, lifecycleConfig=None): class ListBucketsResponse(BaseModel): - allowedAttr = {'buckets': list, 'owner': Owner, 'maxKeys': int, 'marker': BASESTRING, 'isTruncated': bool, 'nextMarker': BASESTRING} + allowedAttr = {'buckets': list, 'owner': Owner, 'maxKeys': int, 'marker': BASESTRING, 'isTruncated': bool, + 'nextMarker': BASESTRING} def __init__(self, buckets=None, owner=None, maxKeys=None, marker=None, isTruncated=None, nextMarker=None): self.buckets = buckets @@ -1247,6 +1266,7 @@ def __init__(self, buckets=None, owner=None, maxKeys=None, marker=None, isTrunca self.isTruncated = isTruncated self.nextMarker = nextMarker + class ListMultipartUploadsResponse(BaseModel): allowedAttr = {'bucket': BASESTRING, 'keyMarker': BASESTRING, 'uploadIdMarker': BASESTRING, 'nextKeyMarker': BASESTRING, 'nextUploadIdMarker': BASESTRING, 'maxUploads': int, @@ -1315,10 +1335,10 @@ def __init__(self, accessContorlAllowOrigin=None, accessContorlAllowHeaders=None class PutContentResponse(BaseModel): allowedAttr = {'storageClass': BASESTRING, 'etag': BASESTRING, 'versionId': BASESTRING, 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, - 'objectUrl': BASESTRING} + 'objectUrl': BASESTRING, "crc64": BASESTRING} def __init__(self, storageClass=None, etag=None, versionId=None, sseKms=None, sseKmsKey=None, - sseC=None, sseCKeyMd5=None, objectUrl=None): + sseC=None, sseCKeyMd5=None, objectUrl=None, crc64=None): self.storageClass = storageClass self.etag = etag self.versionId = versionId @@ -1327,15 +1347,16 @@ def __init__(self, storageClass=None, etag=None, versionId=None, sseKms=None, ss self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 self.objectUrl = objectUrl + self.crc64 = crc64 class AppendObjectResponse(BaseModel): allowedAttr = {'storageClass': BASESTRING, 'etag': BASESTRING, 'nextPosition': LONG, 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, - 'objectUrl': BASESTRING} + 'objectUrl': BASESTRING, "crc64": BASESTRING} def __init__(self, storageClass=None, etag=None, nextPosition=None, sseKms=None, sseKmsKey=None, - sseC=None, sseCKeyMd5=None, objectUrl=None): + sseC=None, sseCKeyMd5=None, objectUrl=None, crc64=None): self.storageClass = storageClass self.etag = etag self.nextPosition = nextPosition @@ -1344,18 +1365,20 @@ def __init__(self, storageClass=None, etag=None, nextPosition=None, sseKms=None, self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 self.objectUrl = objectUrl + self.crc64 = crc64 class UploadPartResponse(BaseModel): allowedAttr = {'etag': BASESTRING, 'sseKms': BASESTRING, 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, - 'sseCKeyMd5': BASESTRING} + 'sseCKeyMd5': BASESTRING, "crc64": BASESTRING} - def __init__(self, etag=None, sseKms=None, sseKmsKey=None, sseC=None, sseCKeyMd5=None): + def __init__(self, etag=None, sseKms=None, sseKmsKey=None, sseC=None, sseCKeyMd5=None, crc64=None): self.etag = etag self.sseKms = sseKms self.sseKmsKey = sseKmsKey self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 + self.crc64 = crc64 class GetBucketRequestPaymentResponse(BaseModel): @@ -1418,14 +1441,14 @@ class ObjectStream(BaseModel): 'contentType': BASESTRING, 'expires': BASESTRING, 'websiteRedirectLocation': BASESTRING, 'lastModified': BASESTRING, 'etag': BASESTRING, 'versionId': BASESTRING, 'restore': BASESTRING, 'expiration': BASESTRING, 'sseKms': BASESTRING, - 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING} + 'sseKmsKey': BASESTRING, 'sseC': BASESTRING, 'sseCKeyMd5': BASESTRING, "crc64": BASESTRING} def __init__(self, response=None, buffer=None, size=None, url=None, deleteMarker=None, storageClass=None, accessContorlAllowOrigin=None, accessContorlAllowHeaders=None, accessContorlAllowMethods=None, accessContorlExposeHeaders=None, accessContorlMaxAge=None, contentLength=None, cacheControl=None, contentDisposition=None, contentEncoding=None, contentLanguage=None, contentType=None, expires=None, websiteRedirectLocation=None, lastModified=None, etag=None, versionId=None, restore=None, - expiration=None, sseKms=None, sseKmsKey=None, sseC=None, sseCKeyMd5=None): + expiration=None, sseKms=None, sseKmsKey=None, sseC=None, sseCKeyMd5=None, crc64=None): self.response = response self.buffer = buffer self.size = size @@ -1454,6 +1477,7 @@ def __init__(self, response=None, buffer=None, size=None, url=None, deleteMarker self.sseKmsKey = sseKmsKey self.sseC = sseC self.sseCKeyMd5 = sseCKeyMd5 + self.crc64 = crc64 class ExtensionHeader(BaseModel): diff --git a/src/obs/util.py b/src/obs/util.py index 7f184df..8e29c8c 100644 --- a/src/obs/util.py +++ b/src/obs/util.py @@ -17,7 +17,7 @@ import json import re -from obs import const, progress +from obs import const, progress, crc64mod if const.IS_PYTHON2: import urllib @@ -430,3 +430,71 @@ def _byteify(data, ignore_dicts=False): for key, value in data.iteritems() } return data + + +class Crc64(object): + _POLY = 0x142F0E1EBA9EA3693 + _XOROUT = 0XFFFFFFFFFFFFFFFF + + def __init__(self, init_crc=0): + self.crc64 = crc64mod.Crc(self._POLY, initCrc=init_crc, rev=True, xorOut=self._XOROUT) + + self.crc64_combineFun = crc64mod.mkCombineFun(self._POLY, initCrc=init_crc, rev=True, xorOut=self._XOROUT) + + def __call__(self, data): + self.update(data) + + def update(self, data): + self.crc64.update(data) + + def combine(self, crc1, crc2, len2): + return self.crc64_combineFun(crc1, crc2, len2) + @property + def crc(self): + return self.crc64.crcValue + + +def calculate_file_crc64(file_name, block_size=64 * 1024, init_crc=0, offset=None, totalCount=None): + readCount = 0 + with open(file_name, 'rb') as f: + if offset: + f.seek(offset) + crc64 = Crc64(init_crc) + while True: + if totalCount is None or totalCount - readCount >= block_size: + readCountOnce = block_size + else: + readCountOnce = totalCount -readCount + data = f.read(readCountOnce) + readCount += readCountOnce + if (totalCount is not None and readCount >= totalCount) or not data: + crc64.update(data) + break + crc64.update(data) + f.close() + return crc64.crc + + + +def calculate_content_crc64(content, block_size=64 * 1024, init_crc=0): + crc64 = Crc64(init_crc) + if hasattr(content, 'read'): + while True: + data = content.read(block_size) + if not data: + break + crc64.update(data) + else: + crc64.update(content) + + return crc64.crc + +def calc_obj_crc_from_parts(parts, init_crc=0): + object_crc = 0 + crc_obj = Crc64(init_crc) + for part in parts: + if not part.crc64 or not part.size: + raise Exception('part {0} has no size or crc64'.format(part.partNum)) + else: + object_crc = crc_obj.combine(object_crc, to_int(part.crc64), part.size) + return object_crc \ No newline at end of file diff --git a/src/tests/test_obs_client.py b/src/tests/test_obs_client.py index 4218cf6..5d1e795 100644 --- a/src/tests/test_obs_client.py +++ b/src/tests/test_obs_client.py @@ -23,7 +23,8 @@ import conftest from obs import CreateBucketHeader, GetObjectHeader, ObsClient, UploadFileHeader, \ Expiration, NoncurrentVersionExpiration, AbortIncompleteMultipartUpload, DateTime, \ - Rule, Lifecycle + Rule, Lifecycle, PutObjectHeader, AppendObjectHeader, AppendObjectContent, util, CompleteMultipartUploadRequest, \ + CompletePart from conftest import test_config @@ -436,88 +437,214 @@ def test_setBucketLifecycle_and_getBucketLifecycle_success(self): def test_setAccessLabel_success(self): client_type, accessLabelClient, obsClient = self.get_client() accessLabelList = ['role_label_01', 'role_label_02'] - obsClient.putContent('pythonsdktestbucket-posix', 'dir1/') - set_al_result = accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', accessLabelList) + obsClient.putContent('accesslabel-posix-bucket', 'dir1/') + set_al_result = accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', accessLabelList) assert set_al_result.status == 204 - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir1/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir1/') def test_setAccessLabel_fail(self): client_type, accessLabelClient, obsClient = self.get_client() - obsClient.putContent('pythonsdktestbucket-posix', 'dir1/') - obsClient.putContent('pythonsdktestbucket-posix', 'file1', content='123') - set_al_result1 = accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', ['role-label-01']) + obsClient.putContent('accesslabel-posix-bucket', 'dir1/') + obsClient.putContent('accesslabel-posix-bucket', 'file1', content='123') + set_al_result1 = accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', ['role-label-01']) assert set_al_result1.status == 400 - set_al_result2 = accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', + set_al_result2 = accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', ['abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ']) assert set_al_result2.status == 400 - set_al_result3 = accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', + set_al_result3 = accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', ["role_label_" + str(i + 1) for i in range(513)]) assert set_al_result3.status == 405 - set_al_result4 = accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'file1', + set_al_result4 = accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'file1', ['role_label_01', 'role_label_02']) assert set_al_result4.status == 405 - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir1/') - obsClient.deleteObject('pythonsdktestbucket-posix', 'file1') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir1/') + obsClient.deleteObject('accesslabel-posix-bucket', 'file1') def test_getAccessLabel_success(self): client_type, accessLabelClient, obsClient = self.get_client() - obsClient.putContent('pythonsdktestbucket-posix', 'dir1/') - obsClient.putContent('pythonsdktestbucket-posix', 'dir2/') - obsClient.putContent('pythonsdktestbucket-posix', 'dir3/') - obsClient.putContent('pythonsdktestbucket-posix', 'dir4/') - accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', ['role_label_01', 'role_label_02']) - get_al_result1 = accessLabelClient.getAccessLabel('pythonsdktestbucket-posix', 'dir1') + obsClient.putContent('accesslabel-posix-bucket', 'dir1/') + obsClient.putContent('accesslabel-posix-bucket', 'dir2/') + obsClient.putContent('accesslabel-posix-bucket', 'dir3/') + obsClient.putContent('accesslabel-posix-bucket', 'dir4/') + accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', ['role_label_01', 'role_label_02']) + get_al_result1 = accessLabelClient.getAccessLabel('accesslabel-posix-bucket', 'dir1') assert get_al_result1.status == 200 assert get_al_result1.body['accesslabel'] == ['role_label_01', 'role_label_02'] - accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir2', + accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir2', ['abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ']) - get_al_result2 = accessLabelClient.getAccessLabel('pythonsdktestbucket-posix', 'dir2') + get_al_result2 = accessLabelClient.getAccessLabel('accesslabel-posix-bucket', 'dir2') assert get_al_result2.status == 200 assert get_al_result2.body['accesslabel'] == ['abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'] - accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir3', + accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir3', ["role_label_" + str(i + 1) for i in range(512)]) - get_al_result3 = accessLabelClient.getAccessLabel('pythonsdktestbucket-posix', 'dir3') + get_al_result3 = accessLabelClient.getAccessLabel('accesslabel-posix-bucket', 'dir3') assert get_al_result3.status == 200 assert get_al_result3.body['accesslabel'] == ["role_label_" + str(i + 1) for i in range(512)] - accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir4', + accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir4', ["abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW" + str(i + 1) for i in range(512)]) - get_al_result4 = accessLabelClient.getAccessLabel('pythonsdktestbucket-posix', 'dir4') + get_al_result4 = accessLabelClient.getAccessLabel('accesslabel-posix-bucket', 'dir4') assert get_al_result4.status == 200 - assert get_al_result4.body['accesslabel'] == ["abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW" + str(i + 1) for - i in range(512)] - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir1/') - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir2/') - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir3/') - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir4/') + assert get_al_result4.body['accesslabel'] == ["abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW" + str(i + 1) + for + i in range(512)] + obsClient.deleteObject('accesslabel-posix-bucket', 'dir1/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir2/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir3/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir4/') def test_getAccessLabel_fail(self): client_type, accessLabelClient, obsClient = self.get_client() - obsClient.putContent('pythonsdktestbucket-posix', 'dir1/') - get_al_result = accessLabelClient.getAccessLabel('pythonsdktestbucket-posix', 'dir1') + obsClient.putContent('accesslabel-posix-bucket', 'dir1/') + get_al_result = accessLabelClient.getAccessLabel('accesslabel-posix-bucket', 'dir1') assert get_al_result.status == 404 - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir1/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir1/') def test_deleteAccessLabel_success(self): client_type, accessLabelClient, obsClient = self.get_client() - obsClient.putContent('pythonsdktestbucket-posix', 'dir1/') - obsClient.putContent('pythonsdktestbucket-posix', 'dir2/') - accessLabelClient.setAccessLabel('pythonsdktestbucket-posix', 'dir1', ['role_label_01', 'role_label_02']) - del_al_result1 = accessLabelClient.deleteAccessLabel('pythonsdktestbucket-posix', 'dir1') - del_al_result2 = accessLabelClient.deleteAccessLabel('pythonsdktestbucket-posix', 'dir2') - del_al_result3 = accessLabelClient.deleteAccessLabel('pythonsdktestbucket-posix', 'dir3') + obsClient.putContent('accesslabel-posix-bucket', 'dir1/') + obsClient.putContent('accesslabel-posix-bucket', 'dir2/') + accessLabelClient.setAccessLabel('accesslabel-posix-bucket', 'dir1', ['role_label_01', 'role_label_02']) + del_al_result1 = accessLabelClient.deleteAccessLabel('accesslabel-posix-bucket', 'dir1') + del_al_result2 = accessLabelClient.deleteAccessLabel('accesslabel-posix-bucket', 'dir2') + del_al_result3 = accessLabelClient.deleteAccessLabel('accesslabel-posix-bucket', 'dir3') assert del_al_result1.status == 204 assert del_al_result2.status == 204 assert del_al_result3.status == 204 - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir1/') - obsClient.deleteObject('pythonsdktestbucket-posix', 'dir2/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir1/') + obsClient.deleteObject('accesslabel-posix-bucket', 'dir2/') def test_deleteAccessLabel_fail(self): client_type, accessLabelClient, obsClient = self.get_client() - obsClient.putContent('pythonsdktestbucket-posix', 'file1', content='123') - del_al_result1 = accessLabelClient.deleteAccessLabel('pythonsdktestbucket-posix', 'file1') + obsClient.putContent('accesslabel-posix-bucket', 'file1', content='123') + del_al_result1 = accessLabelClient.deleteAccessLabel('accesslabel-posix-bucket', 'file1') assert del_al_result1.status == 405 - obsClient.deleteObject('pythonsdktestbucket-posix', 'file1') + obsClient.deleteObject('accesslabel-posix-bucket', 'file1') + + def test_putObject_with_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_crc64_object" + conftest.gen_random_file(object_name, 1024) + crc64 = util.calculate_file_crc64(test_config["path_prefix"] + object_name) + headers = PutObjectHeader() + headers.isAttachCrc64 = True + put_result = crc64Client.putFile(test_config["bucketName"], object_name, + test_config["path_prefix"] + object_name, + headers=headers) + assert put_result.status == 200 + get_result = crc64Client.getObject(test_config["bucketName"], object_name) + assert int(get_result.body.crc64) == crc64 + wrong_crc64 = 123456789 + headers.crc64 = wrong_crc64 + wrong_crc64_result = crc64Client.putFile(test_config["bucketName"], object_name, + test_config["path_prefix"] + object_name, headers=headers) + assert wrong_crc64_result.status == 400 + obsClient.deleteObject(test_config["bucketName"], object_name) + + def test_putEmptyObject_with_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_empty_crc64_object" + content = "" + headers = PutObjectHeader() + headers.isAttachCrc64 = True + put_result = crc64Client.putContent(test_config["bucketName"], object_name, content, + headers=headers) + assert put_result.status == 200 + get_result = crc64Client.getObject(test_config["bucketName"], object_name) + assert get_result.body.crc64 == '0' + obsClient.deleteObject(test_config["bucketName"], object_name) + + def test_putObject_to_posixBucket_with_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + bucketName = 'accesslabel-posix-bucket' + object_name = "test_crc64_object" + conftest.gen_random_file(object_name, 1024) + headers = PutObjectHeader() + headers.isAttachCrc64 = True + put_result = crc64Client.putFile(bucketName, object_name, test_config["path_prefix"] + object_name, + headers=headers) + assert put_result.status == 405 + + def test_uploadPart_with_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_crc64_object" + conftest.gen_random_file(object_name, 1024) + crc64 = util.calculate_file_crc64(test_config["path_prefix"] + object_name) + + init_result = crc64Client.initiateMultipartUpload(test_config["bucketName"], object_name) + uploadId = init_result.body.uploadId + wrong_crc64_result = crc64Client.uploadPart(test_config["bucketName"], object_name, 1, uploadId, + test_config["path_prefix"] + object_name, True, 1024 * 1024, + crc64=123456789) + assert wrong_crc64_result.status == 400 + + put_result = crc64Client.uploadPart(test_config["bucketName"], object_name, 1, uploadId, + test_config["path_prefix"] + object_name, True, 1024 * 1024, + isAttachCrc64=True) + assert put_result.status == 200 + part1 = CompletePart(partNum=1, etag=put_result.body.etag, crc64=put_result.body.crc64, size=1024 * 1024) + completeMultipartUploadRequest = CompleteMultipartUploadRequest(parts=[part1]) + complete_result = crc64Client.completeMultipartUpload(test_config["bucketName"], object_name, uploadId, + completeMultipartUploadRequest, isAttachCrc64=True) + assert int(complete_result.body.crc64) == crc64 + get_result = crc64Client.getObject(test_config["bucketName"], object_name) + assert int(get_result.body.crc64) == crc64 + obsClient.deleteObject(test_config["bucketName"], object_name) + + def test_appendObject_with_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_crc64_object" + object_name2 = "test_crc64_object2" + headers = AppendObjectHeader() + headers.isAttachCrc64 = True + content = AppendObjectContent() + content.content = 'Hello OBS' + content.position = 0 + crc64 = util.calculate_content_crc64(content.content) + put_result = crc64Client.appendObject(test_config["bucketName"], object_name, + content, headers=headers) + assert put_result.status == 200 + get_result = crc64Client.getObject(test_config["bucketName"], object_name) + assert int(get_result.body.crc64) == crc64 + wrong_crc64 = 123456789 + headers.crc64 = wrong_crc64 + wrong_crc64_result = crc64Client.appendObject(test_config["bucketName"], object_name2, + content, headers=headers) + assert wrong_crc64_result.status == 400 + obsClient.deleteObject(test_config["bucketName"], object_name) + + def test_getObjectMetadata_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_crc64_object" + conftest.gen_random_file(object_name, 1024) + crc64 = util.calculate_file_crc64(test_config["path_prefix"] + object_name) + headers = PutObjectHeader() + headers.isAttachCrc64 = True + put_result = crc64Client.putFile(test_config["bucketName"], object_name, + test_config["path_prefix"] + object_name, headers=headers) + assert put_result.status == 200 + get_result = crc64Client.getObjectMetadata(test_config["bucketName"], object_name) + assert int(get_result.body.crc64) == crc64 + + obsClient.deleteObject(test_config["bucketName"], object_name) + + def test_getRangeObject_crc64(self): + client_type, crc64Client, obsClient = self.get_client() + object_name = "test_crc64_object" + conftest.gen_random_file(object_name, 1024) + crc64 = util.calculate_file_crc64(test_config["path_prefix"] + object_name) + headers = PutObjectHeader() + headers.isAttachCrc64 = True + put_result = crc64Client.putFile(test_config["bucketName"], object_name, + test_config["path_prefix"] + object_name, headers=headers) + assert put_result.status == 200 + get_headers = GetObjectHeader() + get_headers.range = '0-512' + get_result = crc64Client.getObject(test_config["bucketName"], object_name, headers=get_headers) + assert int(get_result.body.crc64) == crc64 + + obsClient.deleteObject(test_config["bucketName"], object_name) + if __name__ == "__main__": pytest.main(["-v", 'test_obs_client.py::TestOBSClient::test_uploadFile_with_metadata'])