diff --git a/Sources/Alloy/Core/Extensions/Metal/MTLPixelFormat+Extensions.swift b/Sources/Alloy/Core/Extensions/Metal/MTLPixelFormat+Extensions.swift index 6ae012b..58bdd4d 100644 --- a/Sources/Alloy/Core/Extensions/Metal/MTLPixelFormat+Extensions.swift +++ b/Sources/Alloy/Core/Extensions/Metal/MTLPixelFormat+Extensions.swift @@ -6,6 +6,7 @@ public extension MTLPixelFormat { } var size: Int? { + #if os(iOS) && !targetEnvironment(macCatalyst) switch self { case .a8Unorm, .r8Unorm, .r8Snorm, .r8Uint, .r8Sint, .stencil8, .r8Unorm_srgb: return 1 @@ -13,6 +14,29 @@ public extension MTLPixelFormat { .r16Sint, .r16Float, .rg8Unorm, .rg8Snorm, .rg8Uint, .rg8Sint, .depth16Unorm, .rg8Unorm_srgb: return 2 + case .r32Uint, .r32Sint, .r32Float, + .rg16Unorm, .rg16Snorm, .rg16Uint, + .rg16Sint, .rg16Float, .rgba8Unorm, + .rgba8Unorm_srgb, .rgba8Snorm, .rgba8Uint, + .rgba8Sint, .bgra8Unorm, .bgra8Unorm_srgb, + .rgb10a2Unorm, .rgb10a2Uint, .rg11b10Float, + .rgb9e5Float, .bgr10a2Unorm, .gbgr422, + .bgrg422, .depth32Float, .bgr10_xr_srgb, .bgr10_xr: return 4 + case .rg32Uint, .rg32Sint, .rg32Float, + .rgba16Unorm, .rgba16Snorm, .rgba16Uint, + .rgba16Sint, .rgba16Float, .depth32Float_stencil8, .x32_stencil8, + .bgra10_xr, .bgra10_xr_srgb: return 8 + case .rgba32Uint, .rgba32Sint, .rgba32Float: return 16 + default: return nil + } + #elseif os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + switch self { + case .a8Unorm, .r8Unorm, .r8Snorm, + .r8Uint, .r8Sint, .stencil8: return 1 + case .r16Unorm, .r16Snorm, .r16Uint, + .r16Sint, .r16Float, .rg8Unorm, + .rg8Snorm, .rg8Uint, .rg8Sint, + .depth16Unorm: return 2 case .r32Uint, .r32Sint, .r32Float, .rg16Unorm, .rg16Snorm, .rg16Uint, .rg16Sint, .rg16Float, .rgba8Unorm, @@ -21,19 +45,17 @@ public extension MTLPixelFormat { .rgb10a2Unorm, .rgb10a2Uint, .rg11b10Float, .rgb9e5Float, .bgr10a2Unorm, .gbgr422, .bgrg422, .depth32Float, .depth24Unorm_stencil8, - .x24_stencil8, .bgr10_xr_srgb, .bgr10_xr: return 4 + .x24_stencil8: return 4 case .rg32Uint, .rg32Sint, .rg32Float, .rgba16Unorm, .rgba16Snorm, .rgba16Uint, .rgba16Sint, .rgba16Float, .bc1_rgba, - .bc1_rgba_srgb, .depth32Float_stencil8, .x32_stencil8, - .bgra10_xr, .bgra10_xr_srgb: return 8 + .bc1_rgba_srgb, .depth32Float_stencil8, .x32_stencil8: return 8 case .rgba32Uint, .rgba32Sint, .rgba32Float, .bc2_rgba, .bc2_rgba_srgb, .bc3_rgba, .bc3_rgba_srgb: return 16 - default: - // TODO: Finish bc4-bc7 - return nil + default: return nil } + #endif } var isOrdinary8Bit: Bool { @@ -175,11 +197,18 @@ public extension MTLPixelFormat { var isCompressed: Bool { #if os(iOS) && !targetEnvironment(macCatalyst) - return self.isPVRTC - || self.isEAC - || self.isETC - || self.isASTC - || self.isHDRASTC + if #available(iOS 13.0, *) { + return self.isPVRTC + || self.isEAC + || self.isETC + || self.isASTC + || self.isHDRASTC + } else { + return self.isPVRTC + || self.isEAC + || self.isETC + || self.isASTC + } #elseif os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) return self.isS3TC || self.isRGTC @@ -210,15 +239,13 @@ public extension MTLPixelFormat { } } + @available(iOS 13.0, *) var isHDRASTC: Bool { switch self { - case .astc_4x4_hdr, - .astc_5x4_hdr, .astc_5x5_hdr, - .astc_6x5_hdr, .astc_6x6_hdr, - .astc_8x5_hdr, .astc_8x6_hdr, .astc_8x8_hdr, - .astc_10x5_hdr, .astc_10x6_hdr, .astc_10x8_hdr, .astc_10x10_hdr, + case .astc_4x4_hdr, .astc_5x4_hdr, .astc_5x5_hdr, .astc_6x5_hdr, .astc_6x6_hdr, .astc_8x5_hdr, + .astc_8x6_hdr, .astc_8x8_hdr, .astc_10x5_hdr, .astc_10x6_hdr, .astc_10x8_hdr, .astc_10x10_hdr, .astc_12x10_hdr, .astc_12x12_hdr: - return true + return true default: return false } } @@ -282,10 +309,16 @@ public extension MTLPixelFormat { } var isDepth: Bool { - switch self { - case .depth16Unorm, .depth32Float: - return true - default: return false + if #available(iOS 13.0, *) { + switch self { + case .depth16Unorm, .depth32Float: return true + default: return false + } + } else { + switch self { + case .depth32Float: return true + default: return false + } } } diff --git a/Sources/Alloy/Core/Extensions/Metal/MTLTexture+Extensions.swift b/Sources/Alloy/Core/Extensions/Metal/MTLTexture+Extensions.swift index edd56ef..af95d64 100644 --- a/Sources/Alloy/Core/Extensions/Metal/MTLTexture+Extensions.swift +++ b/Sources/Alloy/Core/Extensions/Metal/MTLTexture+Extensions.swift @@ -20,8 +20,7 @@ public extension MTLTexture { let rowBytes = self.width let length = rowBytes * self.height - let rgbaBytes = UnsafeMutableRawPointer.allocate(byteCount: length, - alignment: MemoryLayout.alignment) + let rgbaBytes = UnsafeMutablePointer.allocate(capacity: length) defer { rgbaBytes.deallocate() } self.getBytes(rgbaBytes, bytesPerRow: rowBytes, @@ -33,7 +32,7 @@ public extension MTLTexture { ? CGImageAlphaInfo.alphaOnly.rawValue : CGImageAlphaInfo.none.rawValue) guard let data = CFDataCreate(nil, - rgbaBytes.assumingMemoryBound(to: UInt8.self), + rgbaBytes, length), let dataProvider = CGDataProvider(data: data), let cgImage = CGImage(width: self.width, @@ -54,25 +53,20 @@ public extension MTLTexture { // read texture as byte array let rowBytes = self.width * 4 let length = rowBytes * self.height - - let bgraBytes = UnsafeMutableRawPointer.allocate(byteCount: length, - alignment: MemoryLayout.alignment) - defer { bgraBytes.deallocate() } - + let bgraBytes = UnsafeMutablePointer.allocate(capacity: length) + let rgbaBytes = UnsafeMutablePointer.allocate(capacity: length) + defer { bgraBytes.deallocate(); rgbaBytes.deallocate() } self.getBytes(bgraBytes, bytesPerRow: rowBytes, from: self.region, mipmapLevel: 0) // use Accelerate framework to convert from BGRA to RGBA + var bgraBuffer = vImage_Buffer(data: bgraBytes, height: vImagePixelCount(self.height), width: vImagePixelCount(self.width), rowBytes: rowBytes) - - let rgbaBytes = UnsafeMutableRawPointer.allocate(byteCount: length, - alignment: MemoryLayout.alignment) - defer { rgbaBytes.deallocate() } var rgbaBuffer = vImage_Buffer(data: rgbaBytes, height: vImagePixelCount(self.height), width: vImagePixelCount(self.width), @@ -86,7 +80,7 @@ public extension MTLTexture { let colorScape = colorSpace ?? CGColorSpaceCreateDeviceRGB() let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue) guard let data = CFDataCreate(nil, - rgbaBytes.assumingMemoryBound(to: UInt8.self), + rgbaBytes, length), let dataProvider = CGDataProvider(data: data), let cgImage = CGImage(width: self.width, @@ -107,10 +101,8 @@ public extension MTLTexture { let rowBytes = self.width * 4 let length = rowBytes * self.height - let rgbaBytes = UnsafeMutableRawPointer.allocate(byteCount: length, - alignment: MemoryLayout.alignment) + let rgbaBytes = UnsafeMutablePointer.allocate(capacity: length) defer { rgbaBytes.deallocate() } - self.getBytes(rgbaBytes, bytesPerRow: rowBytes, from: self.region, @@ -119,7 +111,7 @@ public extension MTLTexture { let colorScape = colorSpace ?? CGColorSpaceCreateDeviceRGB() let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue) guard let data = CFDataCreate(nil, - rgbaBytes.assumingMemoryBound(to: UInt8.self), + rgbaBytes, length), let dataProvider = CGDataProvider(data: data), let cgImage = CGImage(width: self.width, diff --git a/Sources/Alloy/Core/Extensions/MetalPerformanceShaders/MPSImage+Extensions.swift b/Sources/Alloy/Core/Extensions/MetalPerformanceShaders/MPSImage+Extensions.swift index ffd4c03..263dc75 100644 --- a/Sources/Alloy/Core/Extensions/MetalPerformanceShaders/MPSImage+Extensions.swift +++ b/Sources/Alloy/Core/Extensions/MetalPerformanceShaders/MPSImage+Extensions.swift @@ -9,36 +9,53 @@ public extension MPSImage { func toFloatArray() -> [Float]? { switch self.pixelFormat { case .r16Float, .rg16Float, .rgba16Float: - guard var outputFloat16 = self.convert(initial: Float16(0)) else { return nil } - return float16to32(&outputFloat16, count: outputFloat16.count) + if #available(iOS 14.0, macOS 11.0, macCatalyst 14.5, *), + let float16Array: [Swift.Float16] = self.toArray() { + return float16Array.map(Float.init) + } else if var float16Array: [Float16] = self.toArray() { + return float16to32(&float16Array, + count: float16Array.count) + } else { return nil } case .r32Float, .rg32Float, .rgba32Float, .depth32Float: - return self.convert(initial: Float(0)) + return self.toArray() case .invalid: return nil default: return nil } } - private func convert(initial: T) -> [T]? { - guard self.texture.isAccessibleOnCPU else { return nil } + private func toArray() -> [T]? { + guard self.texture.isAccessibleOnCPU + else { return nil } let numSlices = (self.featureChannels + 3) / 4 /// If the number of channels is not a multiple of 4, we may need to add /// padding. For 1 and 2 channels we don't need padding. - let channelsPlusPadding = (self.featureChannels < 3) ? self.featureChannels : numSlices * 4 + let channelsPlusPadding = (self.featureChannels < 3) + ? self.featureChannels + : numSlices * 4 /// How many elements we need to copy over from each pixel in a slice. /// For 1 channel it's just 1 element (R); for 2 channels it is 2 elements /// (R+G), and for any other number of channels it is 4 elements (RGBA). - let numComponents = (self.featureChannels < 3) ? self.featureChannels : 4 + let numComponents = (self.featureChannels < 3) + ? self.featureChannels + : 4 /// Allocate the memory for the array. If batching is used, then we need to /// copy numSlices slices for each image in the batch. - let count = self.width * self.height * channelsPlusPadding * self.numberOfImages - var output = [T](repeating: initial, count: count) + let count = self.width + * self.height + * channelsPlusPadding + * self.numberOfImages - let region = MTLRegion(origin: MTLOrigin(x: 0, y: 0, z: 0), - size: MTLSize(width: self.width, height: self.height, depth: 1)) + var output = [T](repeating: .zero, + count: count) + + let region = MTLRegion(origin: .zero, + size: .init(width: self.width, + height: self.height, + depth: 1)) for i in 0 ..< numSlices * self.numberOfImages { self.texture.getBytes(&(output[self.width * self.height * numComponents * i]), @@ -48,6 +65,7 @@ public extension MPSImage { mipmapLevel: 0, slice: i) } + return output } } diff --git a/Sources/Alloy/Core/Float16.swift b/Sources/Alloy/Core/Float16.swift index 263b514..3ec3e74 100644 --- a/Sources/Alloy/Core/Float16.swift +++ b/Sources/Alloy/Core/Float16.swift @@ -16,22 +16,29 @@ public typealias Float16 = UInt16 /// - Returns: An array of regular Swift `Float`s. public func float16to32(_ input: UnsafeMutableRawPointer, count: Int) -> [Float]? { - var output = [Float](repeating: 0, + let output = [Float](repeating: 0, count: count) - let status = output.withUnsafeMutableBytes { p -> Int in - var bufferFloat16 = vImage_Buffer(data: input, - height: 1, - width: .init(count), - rowBytes: count * 2) - var bufferFloat32 = vImage_Buffer(data: p.baseAddress, - height: 1, - width: .init(count), - rowBytes: count * 4) - return vImageConvert_Planar16FtoPlanarF(&bufferFloat16, - &bufferFloat32, - 0) + var data: UnsafeMutableRawPointer? = nil + output.withUnsafeBytes { + data = UnsafeMutableRawPointer(mutating: $0.baseAddress) } - return status == kvImageNoError ? output : nil + + var bufferFloat16 = vImage_Buffer(data: input, + height: 1, + width: UInt(count), + rowBytes: count * 2) + var bufferFloat32 = vImage_Buffer(data: data, + height: 1, + width: UInt(count), + rowBytes: count * 4) + + if vImageConvert_Planar16FtoPlanarF(&bufferFloat16, + &bufferFloat32, + 0) != kvImageNoError { + return nil + } + + return output } /// Uses vImage to convert an array of Swift floats into a buffer of float16s. @@ -42,18 +49,26 @@ public func float16to32(_ input: UnsafeMutableRawPointer, /// - Returns: An array of `Float16`s. public func float32to16(_ input: UnsafeMutablePointer, count: Int) -> [Float16]? { - var output = [Float16](repeating: 0, + let output = [Float16](repeating: 0, count: count) - let status = output.withUnsafeMutableBytes { p -> Int in - var bufferFloat32 = vImage_Buffer(data: input, - height: 1, - width: .init(count), - rowBytes: count * 4) - var bufferFloat16 = vImage_Buffer(data: p.baseAddress, - height: 1, - width: .init(count), - rowBytes: count * 2) - return vImageConvert_PlanarFtoPlanar16F(&bufferFloat32, &bufferFloat16, 0) + var data: UnsafeMutableRawPointer? = nil + output.withUnsafeBytes { + data = UnsafeMutableRawPointer(mutating: $0.baseAddress) + } + + var bufferFloat32 = vImage_Buffer(data: input, + height: 1, + width: UInt(count), + rowBytes: count * 4) + var bufferFloat16 = vImage_Buffer(data: data, + height: 1, + width: UInt(count), + rowBytes: count * 2) + + if vImageConvert_PlanarFtoPlanar16F(&bufferFloat32, + &bufferFloat16, + 0) != kvImageNoError { + return nil } - return status == kvImageNoError ? output : nil + return output } diff --git a/Sources/Alloy/ML/ONNXConvolutionPadding.swift b/Sources/Alloy/ML/ONNXConvolutionPadding.swift index 7128330..66e4514 100644 --- a/Sources/Alloy/ML/ONNXConvolutionPadding.swift +++ b/Sources/Alloy/ML/ONNXConvolutionPadding.swift @@ -30,9 +30,9 @@ public typealias Scales = (height: Int, width: Int) } required convenience public init?(coder aDecoder: NSCoder) { - guard - let data = aDecoder.decodeData(), - let other = NSKeyedUnarchiver.unarchiveObject(with: data) as? ONNXConvolutionPadding + guard let data = aDecoder.decodeData(), + let other = try? NSKeyedUnarchiver.unarchivedObject(ofClass: ONNXConvolutionPadding.self, + from: data) else { return nil } self.init(kernel: other.kernel, strides: other.strides, @@ -43,7 +43,8 @@ public typealias Scales = (height: Int, width: Int) } public func encode(with aCoder: NSCoder) { - aCoder.encode(NSKeyedArchiver.archivedData(withRootObject: self)) + try? aCoder.encode(NSKeyedArchiver.archivedData(withRootObject: self, + requiringSecureCoding: false)) } public func paddingMethod() -> MPSNNPaddingMethod {