From 33bb3e9b5e573aff5fc4a55cba4376d71eed794d Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Wed, 24 Jan 2024 22:18:21 +0530 Subject: [PATCH 1/9] Read data asynchronously --- .../semantic_search/frameworks/onnx/onnx_image_encoder.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/services/semantic_search/frameworks/onnx/onnx_image_encoder.dart b/lib/services/semantic_search/frameworks/onnx/onnx_image_encoder.dart index 5325d8e66..5b91eaaee 100644 --- a/lib/services/semantic_search/frameworks/onnx/onnx_image_encoder.dart +++ b/lib/services/semantic_search/frameworks/onnx/onnx_image_encoder.dart @@ -15,7 +15,7 @@ class OnnxImageEncoder { ..setIntraOpNumThreads(1) ..setSessionGraphOptimizationLevel(GraphOptimizationLevel.ortEnableAll); try { - final bytes = File(args["imageModelPath"]).readAsBytesSync(); + final bytes = await File(args["imageModelPath"]).readAsBytes(); final session = OrtSession.fromBuffer(bytes, sessionOptions); _logger.info('image model loaded'); return session.address; @@ -25,10 +25,10 @@ class OnnxImageEncoder { return -1; } - List inferByImage(Map args) { + Future> inferByImage(Map args) async { final runOptions = OrtRunOptions(); //Check the existence of imagePath locally - final rgb = img.decodeImage(File(args["imagePath"]).readAsBytesSync())!; + final rgb = img.decodeImage(await File(args["imagePath"]).readAsBytes())!; final int nx = rgb.width; final int ny = rgb.height; From 6b6eed73a8c996fe646aa8404c7716c1b53b9502 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Jan 2024 21:51:59 +0530 Subject: [PATCH 2/9] added listener to root of app to know when user is interacting with app and when user is not --- lib/app.dart | 63 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/lib/app.dart b/lib/app.dart index 82bb05e46..e3bd1d8a6 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -1,3 +1,4 @@ +import "dart:async"; import 'dart:io'; import 'package:adaptive_theme/adaptive_theme.dart'; @@ -43,6 +44,7 @@ class EnteApp extends StatefulWidget { class _EnteAppState extends State with WidgetsBindingObserver { final _logger = Logger("EnteAppState"); late Locale locale; + late Timer _userInteractionTimer; @override void initState() { @@ -51,6 +53,9 @@ class _EnteAppState extends State with WidgetsBindingObserver { locale = widget.locale; setupIntentAction(); WidgetsBinding.instance.addObserver(this); + _userInteractionTimer = Timer(const Duration(seconds: 2), () { + //start indexing + }); } setLocale(Locale newLocale) { @@ -69,31 +74,44 @@ class _EnteAppState extends State with WidgetsBindingObserver { } } + resetTimer() { + _userInteractionTimer.cancel(); + _userInteractionTimer = Timer(const Duration(seconds: 2), () { + //resume indexing + }); + } + @override Widget build(BuildContext context) { if (Platform.isAndroid || kDebugMode) { - return AdaptiveTheme( - light: lightThemeData, - dark: darkThemeData, - initial: widget.savedThemeMode ?? AdaptiveThemeMode.system, - builder: (lightTheme, dartTheme) => MaterialApp( - title: "ente", - themeMode: ThemeMode.system, - theme: lightTheme, - darkTheme: dartTheme, - home: AppLifecycleService.instance.mediaExtensionAction.action == - IntentAction.view - ? const FileViewer() - : const HomeWidget(), - debugShowCheckedModeBanner: false, - builder: EasyLoading.init(), - locale: locale, - supportedLocales: appSupportedLocales, - localeListResolutionCallback: localResolutionCallBack, - localizationsDelegates: const [ - ...AppLocalizations.localizationsDelegates, - S.delegate, - ], + return Listener( + onPointerDown: (event) { + //pause indexing + resetTimer(); + }, + child: AdaptiveTheme( + light: lightThemeData, + dark: darkThemeData, + initial: widget.savedThemeMode ?? AdaptiveThemeMode.system, + builder: (lightTheme, dartTheme) => MaterialApp( + title: "ente", + themeMode: ThemeMode.system, + theme: lightTheme, + darkTheme: dartTheme, + home: AppLifecycleService.instance.mediaExtensionAction.action == + IntentAction.view + ? const FileViewer() + : const HomeWidget(), + debugShowCheckedModeBanner: false, + builder: EasyLoading.init(), + locale: locale, + supportedLocales: appSupportedLocales, + localeListResolutionCallback: localResolutionCallBack, + localizationsDelegates: const [ + ...AppLocalizations.localizationsDelegates, + S.delegate, + ], + ), ), ); } else { @@ -119,6 +137,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { @override void dispose() { WidgetsBinding.instance.removeObserver(this); + _userInteractionTimer.cancel(); super.dispose(); } From 5d1379c31440bc2393e11ca4e326940855feaa6b Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Jan 2024 21:59:20 +0530 Subject: [PATCH 3/9] call methods resumeIndexing() and pauseIndexing() in SemanticSearchService when user is not interacting and when is interacting --- lib/app.dart | 7 ++++--- lib/services/semantic_search/semantic_search_service.dart | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/app.dart b/lib/app.dart index e3bd1d8a6..e35da65f5 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -13,6 +13,7 @@ import 'package:photos/ente_theme_data.dart'; import "package:photos/generated/l10n.dart"; import "package:photos/l10n/l10n.dart"; import 'package:photos/services/app_lifecycle_service.dart'; +import "package:photos/services/semantic_search/semantic_search_service.dart"; import 'package:photos/services/sync_service.dart'; import 'package:photos/ui/tabs/home_widget.dart'; import "package:photos/ui/viewer/actions/file_viewer.dart"; @@ -54,7 +55,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { setupIntentAction(); WidgetsBinding.instance.addObserver(this); _userInteractionTimer = Timer(const Duration(seconds: 2), () { - //start indexing + SemanticSearchService.instance.resumeIndexing(); }); } @@ -77,7 +78,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { resetTimer() { _userInteractionTimer.cancel(); _userInteractionTimer = Timer(const Duration(seconds: 2), () { - //resume indexing + SemanticSearchService.instance.resumeIndexing(); }); } @@ -86,7 +87,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { if (Platform.isAndroid || kDebugMode) { return Listener( onPointerDown: (event) { - //pause indexing + SemanticSearchService.instance.pauseIndexing(); resetTimer(); }, child: AdaptiveTheme( diff --git a/lib/services/semantic_search/semantic_search_service.dart b/lib/services/semantic_search/semantic_search_service.dart index 37b1901d5..62d20da17 100644 --- a/lib/services/semantic_search/semantic_search_service.dart +++ b/lib/services/semantic_search/semantic_search_service.dart @@ -52,6 +52,10 @@ class SemanticSearchService { get hasInitialized => _hasInitialized; + void resumeIndexing() {} + + void pauseIndexing() {} + Future init({bool shouldSyncImmediately = false}) async { if (!LocalSettings.instance.hasEnabledMagicSearch()) { return; From b6ade70d67e97b0411e53e3486e7070a726cfe03 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Thu, 25 Jan 2024 22:01:40 +0530 Subject: [PATCH 4/9] add debug logs --- lib/app.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/app.dart b/lib/app.dart index e35da65f5..51911ed29 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -55,6 +55,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { setupIntentAction(); WidgetsBinding.instance.addObserver(this); _userInteractionTimer = Timer(const Duration(seconds: 2), () { + debugPrint("user is not interacting with the app"); SemanticSearchService.instance.resumeIndexing(); }); } @@ -78,6 +79,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { resetTimer() { _userInteractionTimer.cancel(); _userInteractionTimer = Timer(const Duration(seconds: 2), () { + debugPrint("user is not interacting with the app"); SemanticSearchService.instance.resumeIndexing(); }); } @@ -88,6 +90,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { return Listener( onPointerDown: (event) { SemanticSearchService.instance.pauseIndexing(); + debugPrint("user is interacting with the app"); resetTimer(); }, child: AdaptiveTheme( From 686ae81cfa7238e3049aabf04e13cc1a284b4c6b Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Thu, 25 Jan 2024 23:57:58 +0530 Subject: [PATCH 5/9] Increase interaction timeout to 5s --- lib/app.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/app.dart b/lib/app.dart index 51911ed29..5de0e3f79 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -78,7 +78,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { resetTimer() { _userInteractionTimer.cancel(); - _userInteractionTimer = Timer(const Duration(seconds: 2), () { + _userInteractionTimer = Timer(const Duration(seconds: 5), () { debugPrint("user is not interacting with the app"); SemanticSearchService.instance.resumeIndexing(); }); From 177464c203cbf5ee0d736490303673e4d24791a2 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Thu, 25 Jan 2024 23:59:28 +0530 Subject: [PATCH 6/9] Pause image embedding computation when user is interacting with the app --- .../semantic_search_service.dart | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/services/semantic_search/semantic_search_service.dart b/lib/services/semantic_search/semantic_search_service.dart index 62d20da17..8699b0e4e 100644 --- a/lib/services/semantic_search/semantic_search_service.dart +++ b/lib/services/semantic_search/semantic_search_service.dart @@ -49,12 +49,21 @@ class SemanticSearchService { Future>? _ongoingRequest; List _cachedEmbeddings = []; PendingQuery? _nextQuery; + Completer _userInteraction = Completer(); get hasInitialized => _hasInitialized; - void resumeIndexing() {} + void resumeIndexing() { + _logger.info("Resuming indexing"); + _userInteraction.complete(); + } - void pauseIndexing() {} + void pauseIndexing() { + if (_userInteraction.isCompleted) { + _logger.info("Pausing indexing"); + _userInteraction = Completer(); + } + } Future init({bool shouldSyncImmediately = false}) async { if (!LocalSettings.instance.hasEnabledMagicSearch()) { @@ -283,6 +292,10 @@ class SemanticSearchService { if (!_frameworkInitialization.isCompleted) { return; } + if (!_userInteraction.isCompleted) { + _logger.info("Waiting for user interactions to stop..."); + await _userInteraction.future; + } try { final thumbnail = await getThumbnailForUploadedFile(file); if (thumbnail == null) { From c2c858448a421bb12becb1aceca4fb00d10fc004 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Fri, 26 Jan 2024 00:14:30 +0530 Subject: [PATCH 7/9] Start with an initial interaction timeout of 10s and then lower it to 5s --- lib/app.dart | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/app.dart b/lib/app.dart index 5de0e3f79..c5907cc14 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -43,6 +43,9 @@ class EnteApp extends StatefulWidget { } class _EnteAppState extends State with WidgetsBindingObserver { + static const initialInteractionTimeout = Duration(seconds: 10); + static const defaultInteractionTimeout = Duration(seconds: 5); + final _logger = Logger("EnteAppState"); late Locale locale; late Timer _userInteractionTimer; @@ -54,10 +57,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { locale = widget.locale; setupIntentAction(); WidgetsBinding.instance.addObserver(this); - _userInteractionTimer = Timer(const Duration(seconds: 2), () { - debugPrint("user is not interacting with the app"); - SemanticSearchService.instance.resumeIndexing(); - }); + _setupInteractionTimer(timeout: initialInteractionTimeout); } setLocale(Locale newLocale) { @@ -76,9 +76,13 @@ class _EnteAppState extends State with WidgetsBindingObserver { } } - resetTimer() { + void _resetTimer() { _userInteractionTimer.cancel(); - _userInteractionTimer = Timer(const Duration(seconds: 5), () { + _setupInteractionTimer(); + } + + void _setupInteractionTimer({Duration timeout = defaultInteractionTimeout}) { + _userInteractionTimer = Timer(timeout, () { debugPrint("user is not interacting with the app"); SemanticSearchService.instance.resumeIndexing(); }); @@ -91,7 +95,7 @@ class _EnteAppState extends State with WidgetsBindingObserver { onPointerDown: (event) { SemanticSearchService.instance.pauseIndexing(); debugPrint("user is interacting with the app"); - resetTimer(); + _resetTimer(); }, child: AdaptiveTheme( light: lightThemeData, From 775fc68f16fc22c01190adf86259d13498d2421f Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Jan 2024 10:49:39 +0530 Subject: [PATCH 8/9] add listener to root of the app also when (Platform.isAndroid || kDebugMode) is false --- lib/app.dart | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/app.dart b/lib/app.dart index c5907cc14..42de3e7ee 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -123,21 +123,28 @@ class _EnteAppState extends State with WidgetsBindingObserver { ), ); } else { - return MaterialApp( - title: "ente", - themeMode: ThemeMode.system, - theme: lightThemeData, - darkTheme: darkThemeData, - home: const HomeWidget(), - debugShowCheckedModeBanner: false, - builder: EasyLoading.init(), - locale: locale, - supportedLocales: appSupportedLocales, - localeListResolutionCallback: localResolutionCallBack, - localizationsDelegates: const [ - ...AppLocalizations.localizationsDelegates, - S.delegate, - ], + return Listener( + onPointerDown: (event) { + SemanticSearchService.instance.pauseIndexing(); + debugPrint("user is interacting with the app"); + _resetTimer(); + }, + child: MaterialApp( + title: "ente", + themeMode: ThemeMode.system, + theme: lightThemeData, + darkTheme: darkThemeData, + home: const HomeWidget(), + debugShowCheckedModeBanner: false, + builder: EasyLoading.init(), + locale: locale, + supportedLocales: appSupportedLocales, + localeListResolutionCallback: localResolutionCallBack, + localizationsDelegates: const [ + ...AppLocalizations.localizationsDelegates, + S.delegate, + ], + ), ); } } From 225c4644bbd5751a1d5edc7030522238dd11aace Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 26 Jan 2024 11:05:29 +0530 Subject: [PATCH 9/9] bump up version to 0.8.48 --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index 71a0eed13..dbb54f366 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.47+567 +version: 0.8.48+568 environment: sdk: ">=3.0.0 <4.0.0"