diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/Bytes.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/Bytes.swift index 7195cfa050..af562229f4 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/Bytes.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/Bytes.swift @@ -16,22 +16,22 @@ enum Bytes { case kilobytes(Int) case bytes(Int) - var bytes: Int { + var bytes: UInt64 { switch self { case .terabytes(let tb): - return Int(pow(1_024.0, 4.0)) * tb + return UInt64(pow(1_024.0, 4.0)) * UInt64(tb) case .gigabytes(let gb): - return Int(pow(1_024.0, 3.0)) * gb + return UInt64(pow(1_024.0, 3.0)) * UInt64(gb) case .megabytes(let mb): - return Int(pow(1_024.0, 2.0)) * mb + return UInt64(pow(1_024.0, 2.0)) * UInt64(mb) case .kilobytes(let kb): - return 1_024 * kb + return 1_024 * UInt64(kb) case .bytes(let b): - return b + return UInt64(b) } } - var bits: Int { + var bits: UInt64 { return bytes * 8 } } diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileHandle+UInt64.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileHandle+UInt64.swift new file mode 100644 index 0000000000..be6eb4bb49 --- /dev/null +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileHandle+UInt64.swift @@ -0,0 +1,43 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import Foundation + +extension FileHandle { + /// Reads data synchronously up to the specified number of bytes. + /// - Parameter bytes: The number of bytes to read from the file handle. + /// - Parameter bytesReadLimit: The maximum number of bytes that can be read at a time. Defaults to `Int.max`. + /// - Returns: The data available through the receiver up to a maximum of length bytes, or the maximum size that can be represented by a Data object. + /// - Throws: An error if attempts to determine the file-handle type fail or if attempts to read from the file or channel fail. + func read(bytes: UInt64, bytesReadLimit: Int = Int.max) throws -> Data { + // Read as much as it's possible considering the `bytesReadLimit` maximum + let bytesRead = bytes <= bytesReadLimit ? Int(bytes) : bytesReadLimit + guard var data = try readData(upToCount: bytesRead) else { + // There is no more data to read from the file + return Data() + } + + // If there's remaining bytes to read, do it and append to the current data + let remainingBytes = bytes - UInt64(bytesRead) + if remainingBytes > 0 { + try data.append(read( + bytes: remainingBytes, + bytesReadLimit: bytesReadLimit + )) + } + + return data + } + + private func readData(upToCount length: Int) throws -> Data? { + if #available(iOS 13.4, macOS 10.15.4, tvOS 13.4, *) { + return try read(upToCount: length) + } else { + return readData(ofLength: length) + } + } +} diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileSystem.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileSystem.swift index 3f2bd33ae8..fd7c103414 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileSystem.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/FileSystem.swift @@ -185,7 +185,7 @@ class FileSystem { /// - offset: position to start reading /// - length: length of the part /// - completionHandler: completion handler - func createPartialFile(fileURL: URL, offset: Int, length: Int, completionHandler: @escaping (Result) -> Void) { + func createPartialFile(fileURL: URL, offset: UInt64, length: UInt64, completionHandler: @escaping (Result) -> Void) { // 4.5 MB (1 MB per part) // 1024 1024 1024 1024 512 @@ -198,8 +198,8 @@ class FileSystem { defer { try? fileHandle.close() } - try fileHandle.seek(toOffset: UInt64(offset)) - let data = fileHandle.readData(ofLength: length) + try fileHandle.seek(toOffset: offset) + let data = try fileHandle.read(bytes: length) let fileURL = try self.createTemporaryFile(data: data) completionHandler(.success(fileURL)) } catch { diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StoragePersistableTransferTask.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StoragePersistableTransferTask.swift index 5a1233f208..708dc4d93a 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StoragePersistableTransferTask.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StoragePersistableTransferTask.swift @@ -108,8 +108,8 @@ struct StoragePersistableMultipartUpload: Codable { struct StoragePersistableSubTask: Codable { let uploadId: UploadID let partNumber: PartNumber - let bytes: Int - let bytesTransferred: Int + let bytes: UInt64 + let bytesTransferred: UInt64 let taskIdentifier: TaskIdentifier? // once an UploadPart starts uploading it will have a taskIdentifier let eTag: String? diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageServiceSessionDelegate.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageServiceSessionDelegate.swift index 03da3cc4fe..760940dfc7 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageServiceSessionDelegate.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageServiceSessionDelegate.swift @@ -165,7 +165,7 @@ extension StorageServiceSessionDelegate: URLSessionTaskDelegate { return } - multipartUploadSession.handle(uploadPartEvent: .progressUpdated(partNumber: partNumber, bytesTransferred: Int(bytesSent), taskIdentifier: task.taskIdentifier)) + multipartUploadSession.handle(uploadPartEvent: .progressUpdated(partNumber: partNumber, bytesTransferred: UInt64(bytesSent), taskIdentifier: task.taskIdentifier)) case .upload(let onEvent): let progress = Progress(totalUnitCount: totalBytesExpectedToSend) progress.completedUnitCount = totalBytesSent diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPart.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPart.swift index 8481ca61a1..4e5627e20d 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPart.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPart.swift @@ -28,11 +28,11 @@ let minimumPartCount = 1 let maximumPartCount = 10_000 enum StorageUploadPart { - case pending(bytes: Int) - case queued(bytes: Int) - case inProgress(bytes: Int, bytesTransferred: Int, taskIdentifier: TaskIdentifier) - case failed(bytes: Int, bytesTransferred: Int, error: Error) - case completed(bytes: Int, eTag: String) + case pending(bytes: UInt64) + case queued(bytes: UInt64) + case inProgress(bytes: UInt64, bytesTransferred: UInt64, taskIdentifier: TaskIdentifier) + case failed(bytes: UInt64, bytesTransferred: UInt64, error: Error) + case completed(bytes: UInt64, eTag: String) var isPending: Bool { if case .pending = self { @@ -90,8 +90,8 @@ enum StorageUploadPart { return result } - var bytes: Int { - let result: Int + var bytes: UInt64 { + let result: UInt64 switch self { case .pending(let bytes), .queued(let bytes): result = bytes @@ -106,8 +106,8 @@ enum StorageUploadPart { return result } - var bytesTransferred: Int { - let result: Int + var bytesTransferred: UInt64 { + let result: UInt64 switch self { case .pending, .queued, .failed: result = 0 @@ -157,7 +157,7 @@ struct StorageUploadPartSize { case exceedsSupportedFileSize case exceedsMaximumObjectSize } - let size: Int + let size: UInt64 static let `default`: StorageUploadPartSize = StorageUploadPartSize() @@ -167,7 +167,7 @@ struct StorageUploadPartSize { /// Creates custom part size in bytes. Throws if file part is invalid. /// - Parameter size: part size - init(size: Int) throws { + init(size: UInt64) throws { if size < minimumPartSize { throw Failure.belowMinimumPartSize } else if size > maximumPartSize { @@ -215,8 +215,8 @@ struct StorageUploadPartSize { } } - func offset(for partNumber: PartNumber) -> Int { - let result = (partNumber - 1) * size + func offset(for partNumber: PartNumber) -> UInt64 { + let result = UInt64(partNumber - 1) * size return result } @@ -242,7 +242,7 @@ extension Array where Element == StorageUploadPart { throw Failure.partCountOverUpperLimit } - let remainingBytes = Int(fileSize % UInt64(size)) + let remainingBytes = fileSize % size logger.debug("count = \(count), remainingBytes = \(remainingBytes), size = \(size), totalBytes = \(fileSize)") self.init(repeating: .pending(bytes: size), count: count) @@ -298,13 +298,13 @@ extension Sequence where Element == StorageUploadPart { filter { $0.completed } } - var totalBytes: Int { + var totalBytes: UInt64 { reduce(into: 0) { result, part in result += part.bytes } } - var bytesTransferred: Int { + var bytesTransferred: UInt64 { reduce(into: 0) { result, part in result += part.bytesTransferred } diff --git a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPartEvent.swift b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPartEvent.swift index 3e926a0359..7e0981bf59 100644 --- a/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPartEvent.swift +++ b/AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Support/Internal/StorageUploadPartEvent.swift @@ -10,7 +10,7 @@ import Foundation enum StorageUploadPartEvent { case queued(partNumber: PartNumber) case started(partNumber: PartNumber, taskIdentifier: TaskIdentifier) - case progressUpdated(partNumber: PartNumber, bytesTransferred: Int, taskIdentifier: TaskIdentifier) + case progressUpdated(partNumber: PartNumber, bytesTransferred: UInt64, taskIdentifier: TaskIdentifier) case completed(partNumber: PartNumber, eTag: String, taskIdentifier: TaskIdentifier) case failed(partNumber: PartNumber, error: Error) diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileHandleTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileHandleTests.swift new file mode 100644 index 0000000000..8d12cca7af --- /dev/null +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileHandleTests.swift @@ -0,0 +1,47 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import XCTest +@testable import AWSS3StoragePlugin + +class FileHandleTests: XCTestCase { + + /// Given: A FileHandle and a file + /// When: `read(bytes:bytesReadLimit)` is invoked with `bytesReadLimit` being lower than `bytes` + /// Then: Only `bytesReadLimit` bytes will be read at a time, but all `bytes` will be read and returned + func testRead_withBytesHigherThanLimit_shouldSucceedByReadingMultipleTimes() throws { + let sourceString = "012345678910" // 11 bytes + let sourceData = sourceString.data(using: .utf8)! + let sourceFile = try createFile(from: sourceData) + XCTAssertEqual(try StorageRequestUtils.getSize(sourceFile), UInt64(sourceString.count)) + + let fileSystem = FileSystem() + let bytesReadLimit = 2 + + let fileHandle = try FileHandle(forReadingFrom: sourceFile) + let firstPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit) + let firstPartString = String(decoding: firstPartData, as: UTF8.self) + XCTAssertEqual(firstPartString, "01234") // i.e. the first 5 bytes + + let secondPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit) + let secondPartString = String(decoding: secondPartData, as: UTF8.self) + XCTAssertEqual(secondPartString, "56789") // i.e. the second 5 bytes + + let thirdPartData = try fileHandle.read(bytes: 5, bytesReadLimit: bytesReadLimit) + let thirdPartString = String(decoding: thirdPartData, as: UTF8.self) + XCTAssertEqual(thirdPartString, "10") // i.e. the remaining bytes + + try FileManager.default.removeItem(at: sourceFile) + } + + private func createFile(from data: Data) throws -> URL { + let fileUrl = try FileManager.default.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + .appendingPathComponent("\(UUID().uuidString).tmp") + try data.write(to: fileUrl, options: .atomic) + return fileUrl + } +} diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileSystemTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileSystemTests.swift index 2ccce7c3ab..7c573bd843 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileSystemTests.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/FileSystemTests.swift @@ -18,7 +18,7 @@ extension FileSystem { /// - Parameter bytes: bytes /// - Returns: random data func randomData(bytes: Bytes) -> Data { - let count = bytes.bytes + let count = Int(bytes.bytes) var bytes = [Int8](repeating: 0, count: count) // Fill bytes with secure random data let status = SecRandomCopyBytes( @@ -174,7 +174,7 @@ class FileSystemTests: XCTestCase { defer { fs.removeFileIfExists(fileURL: fileURL) } - var offset = 0 + var offset: UInt64 = 0 var step: ((Int) -> Void)? let queue = DispatchQueue(label: "done-count-queue") @@ -190,7 +190,7 @@ class FileSystemTests: XCTestCase { let part = parts[index] print("Creating partial file [\(index)]") - fs.createPartialFile(fileURL: fileURL, offset: offset, length: part.count) { result in + fs.createPartialFile(fileURL: fileURL, offset: offset, length: UInt64(part.count)) { result in do { let partFileURL = try result.get() let fileContents = try String(contentsOf: partFileURL) @@ -210,7 +210,7 @@ class FileSystemTests: XCTestCase { XCTFail("Failed to create partial file: \(error)") } } - offset += part.count + offset += UInt64(part.count) step?(index + 1) } diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/MockMultipartUploadClient.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/MockMultipartUploadClient.swift index 2800a2bb73..49a4bb3265 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/MockMultipartUploadClient.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/MockMultipartUploadClient.swift @@ -21,7 +21,7 @@ class MockMultipartUploadClient: StorageMultipartUploadClient { var didCreate: ((StorageMultipartUploadSession) -> Void)? var didStartPartUpload: ((StorageMultipartUploadSession, PartNumber) -> Void)? - var didTransferBytesForPartUpload: ((StorageMultipartUploadSession, PartNumber, Int) -> Void)? + var didTransferBytesForPartUpload: ((StorageMultipartUploadSession, PartNumber, UInt64) -> Void)? var shouldFailPartUpload: ((StorageMultipartUploadSession, PartNumber) -> Bool)? var didCompletePartUpload: ((StorageMultipartUploadSession, PartNumber, String, TaskIdentifier) -> Void)? var didFailPartUpload: ((StorageMultipartUploadSession, PartNumber, Error) -> Void)? diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageMultipartUploadSessionTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageMultipartUploadSessionTests.swift index dad5a0a531..1ae2882915 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageMultipartUploadSessionTests.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageMultipartUploadSessionTests.swift @@ -321,7 +321,7 @@ class StorageMultipartUploadSessionTests: XCTestCase { // MARK: - Private - private func createFile() throws -> URL { - let size = minimumPartSize + let size = Int(minimumPartSize) let parts: [String] = [ Array(repeating: "a", count: size).joined(), Array(repeating: "b", count: size).joined(), diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageTransferDatabaseTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageTransferDatabaseTests.swift index 15719cde1c..884d4741c1 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageTransferDatabaseTests.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageTransferDatabaseTests.swift @@ -196,11 +196,11 @@ class StorageTransferDatabaseTests: XCTestCase { subTask.uploadPart = .pending(bytes: Bytes.megabytes(5).bytes) if index == 0 { parts[index] = .inProgress(bytes: part.bytes, - bytesTransferred: Int(Double(part.bytes) * 0.75), + bytesTransferred: UInt64(Double(part.bytes) * 0.75), taskIdentifier: sessionTask.taskIdentifier) } else if index == 1 { parts[index] = .inProgress(bytes: part.bytes, - bytesTransferred: Int(Double(part.bytes) * 0.25), + bytesTransferred: UInt64(Double(part.bytes) * 0.25), taskIdentifier: sessionTask.taskIdentifier) } } else { diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartSizeTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartSizeTests.swift index c73c771480..2fb9d9429a 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartSizeTests.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartSizeTests.swift @@ -48,7 +48,7 @@ class StorageUploadPartSizeTests: XCTestCase { func testUploadPartSizeForLargeValidFile() throws { // use a file size which requires increasing from minimum part size - let fileSize = UInt64(minimumPartSize * maximumPartCount * 10) + let fileSize = minimumPartSize * UInt64(maximumPartCount) * 10 let partSize = assertNoThrow(try StorageUploadPartSize(fileSize: fileSize)) XCTAssertNotNil(partSize) if let partSize = partSize, @@ -62,7 +62,7 @@ class StorageUploadPartSizeTests: XCTestCase { func testUploadPartSizeForSuperCrazyBigFile() throws { // use the maximum object size / max part count - let fileSize = UInt64(maximumObjectSize / maximumPartCount) + let fileSize = maximumObjectSize / UInt64(maximumPartCount) let partSize = assertNoThrow(try StorageUploadPartSize(fileSize: fileSize)) XCTAssertNotNil(partSize) if let partSize = partSize, diff --git a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartTests.swift b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartTests.swift index 8d7433563a..73bad639ca 100644 --- a/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartTests.swift +++ b/AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Support/Internal/StorageUploadPartTests.swift @@ -15,9 +15,9 @@ class StorageUploadPartTests: XCTestCase { func testUploadPartCreation() throws { // Creates an array of upload parts with 21 parts // where the last part is 512 bytes. - let lastPartSize = 512 + let lastPartSize: UInt64 = 512 let partSize: StorageUploadPartSize = .default - let fileSize: UInt64 = UInt64(partSize.size * 20 + lastPartSize) + let fileSize = partSize.size * 20 + lastPartSize let parts = try StorageUploadParts(fileSize: fileSize, partSize: partSize) XCTAssertEqual(parts.count, 21) XCTAssertEqual(parts.pending.count, parts.count) @@ -43,8 +43,8 @@ class StorageUploadPartTests: XCTestCase { XCTAssertEqual(parts.inProgress.count, parts.count) XCTAssertEqual(parts.failed.count, 0) XCTAssertEqual(parts.completed.count, 0) - XCTAssertEqual(parts.totalBytes, 100 * parts.count) - XCTAssertEqual(parts.bytesTransferred, 50 * parts.count) + XCTAssertEqual(parts.totalBytes, UInt64(100 * parts.count)) + XCTAssertEqual(parts.bytesTransferred, UInt64(50 * parts.count)) XCTAssertEqual(parts.percentTransferred, 0.5) }