From 1580b37f94eafcab6f3ff3b891269865e9481c12 Mon Sep 17 00:00:00 2001 From: Balazs Perlaki-Horvath Date: Tue, 4 Jun 2024 20:42:45 +0200 Subject: [PATCH 1/4] Split content data and metadata, use data stream to write --- Model/ZimFileService/ZimFileService.mm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Model/ZimFileService/ZimFileService.mm b/Model/ZimFileService/ZimFileService.mm index 767c3326e..978d91912 100644 --- a/Model/ZimFileService/ZimFileService.mm +++ b/Model/ZimFileService/ZimFileService.mm @@ -198,8 +198,7 @@ - (NSDictionary *)getContent:(NSUUID *)zimFileID contentPath:(NSString *)content blob = item.getData(start, fmin(item.getSize() - start, end - start + 1)); } return @{ - @"data": [NSData dataWithBytes:item.getData().data() length:blob.size()], - @"mime": [NSString stringWithUTF8String:item.getMimetype().c_str()], + @"data": [NSData dataWithBytes: blob.data() length:blob.size()], @"start": [NSNumber numberWithUnsignedLongLong:start], @"end": [NSNumber numberWithUnsignedLongLong:start + blob.size() - 1], @"size": [NSNumber numberWithUnsignedLongLong:item.getSize()] From 62264156538a3083e0bbc1d87924025e1ab4645b Mon Sep 17 00:00:00 2001 From: Balazs Perlaki-Horvath Date: Tue, 4 Jun 2024 21:16:28 +0200 Subject: [PATCH 2/4] Use ranges for zim content if larger than 2MB --- Model/Utilities/WebKitHandler.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Model/Utilities/WebKitHandler.swift b/Model/Utilities/WebKitHandler.swift index 5451c263d..60105b556 100644 --- a/Model/Utilities/WebKitHandler.swift +++ b/Model/Utilities/WebKitHandler.swift @@ -88,21 +88,25 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler { let dataProvider: any DataProvider let ranges: [ClosedRange] // the list of ranges we should use to stream data + let size2MB: UInt = 2097152 if metaData.isMediaType, let directAccess = await directAccessInfo(for: url) { dataProvider = ZimDirectContentProvider(directAccess: directAccess, contentSize: metaData.size) ranges = ByteRanges.rangesFor( contentLength: metaData.size, - rangeSize: 2097152 // 2MB + rangeSize: size2MB ) } else { dataProvider = ZimContentProvider(for: url) - // currently using the full range (from 0 to content size) - // this means we read the content in one piece - // once https://github.com/openzim/libzim/issues/886 is fixed - // we can stream compressed data "in chunks" as well - // to be done as part of: https://github.com/kiwix/kiwix-apple/issues/784 - ranges = [0...metaData.size] // it's the full range + // if the data is larger than 2MB, read it "in chunks" + if metaData.size > size2MB { + ranges = ByteRanges.rangesFor( + contentLength: metaData.size, + rangeSize: size2MB + ) + } else { // use the full range and read it in one go + ranges = [0...metaData.size] + } } guard let dataStream = DataStream(dataProvider: dataProvider, ranges: ranges) From c6730875df0825f47a87dd57884ae0a83ebd09de Mon Sep 17 00:00:00 2001 From: Balazs Perlaki-Horvath Date: Tue, 4 Jun 2024 21:51:20 +0200 Subject: [PATCH 3/4] Move DataStream to a new function --- Model/Utilities/WebKitHandler.swift | 52 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/Model/Utilities/WebKitHandler.swift b/Model/Utilities/WebKitHandler.swift index 60105b556..7becdb4d1 100644 --- a/Model/Utilities/WebKitHandler.swift +++ b/Model/Utilities/WebKitHandler.swift @@ -86,31 +86,7 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler { return } - let dataProvider: any DataProvider - let ranges: [ClosedRange] // the list of ranges we should use to stream data - let size2MB: UInt = 2097152 - if metaData.isMediaType, let directAccess = await directAccessInfo(for: url) { - dataProvider = ZimDirectContentProvider(directAccess: directAccess, - contentSize: metaData.size) - ranges = ByteRanges.rangesFor( - contentLength: metaData.size, - rangeSize: size2MB - ) - } else { - dataProvider = ZimContentProvider(for: url) - // if the data is larger than 2MB, read it "in chunks" - if metaData.size > size2MB { - ranges = ByteRanges.rangesFor( - contentLength: metaData.size, - rangeSize: size2MB - ) - } else { // use the full range and read it in one go - ranges = [0...metaData.size] - } - } - - guard let dataStream = DataStream(dataProvider: dataProvider, ranges: ranges) - else { + guard let dataStream = await dataStream(for: url, metaData: metaData) else { sendHTTP404Response(urlSchemeTask, url: url) return } @@ -138,6 +114,32 @@ final class KiwixURLSchemeHandler: NSObject, WKURLSchemeHandler { // MARK: Reading content + private func dataStream(for url: URL, metaData: URLContentMetaData) async -> DataStream? { + let dataProvider: any DataProvider + let ranges: [ClosedRange] // the list of ranges we should use to stream data + let size2MB: UInt = 2097152 // 2MB + if metaData.isMediaType, let directAccess = await directAccessInfo(for: url) { + dataProvider = ZimDirectContentProvider(directAccess: directAccess, + contentSize: metaData.size) + ranges = ByteRanges.rangesFor( + contentLength: metaData.size, + rangeSize: size2MB + ) + } else { + dataProvider = ZimContentProvider(for: url) + // if the data is larger than 2MB, read it "in chunks" + if metaData.size > size2MB { + ranges = ByteRanges.rangesFor( + contentLength: metaData.size, + rangeSize: size2MB + ) + } else { // use the full range and read it in one go + ranges = [0...metaData.size] + } + } + return DataStream(dataProvider: dataProvider, ranges: ranges) + } + private func contentMetaData(for url: URL) async -> URLContentMetaData? { return await withCheckedContinuation { continuation in Task.detached(priority: .utility) { From 8b40d6d34c5224cb55e515524ad5315d002016e5 Mon Sep 17 00:00:00 2001 From: Balazs Perlaki-Horvath Date: Tue, 4 Jun 2024 21:56:14 +0200 Subject: [PATCH 4/4] Remove unused key/value --- Model/ZimFileService/ZimFileService.mm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Model/ZimFileService/ZimFileService.mm b/Model/ZimFileService/ZimFileService.mm index 978d91912..7ba99f2b2 100644 --- a/Model/ZimFileService/ZimFileService.mm +++ b/Model/ZimFileService/ZimFileService.mm @@ -200,8 +200,7 @@ - (NSDictionary *)getContent:(NSUUID *)zimFileID contentPath:(NSString *)content return @{ @"data": [NSData dataWithBytes: blob.data() length:blob.size()], @"start": [NSNumber numberWithUnsignedLongLong:start], - @"end": [NSNumber numberWithUnsignedLongLong:start + blob.size() - 1], - @"size": [NSNumber numberWithUnsignedLongLong:item.getSize()] + @"end": [NSNumber numberWithUnsignedLongLong:start + blob.size() - 1] }; } catch (std::exception) { return nil;