From 73a8259ccb8eaaf63d107b267b51fee991a6ffcd Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 27 Sep 2023 16:03:11 -0400 Subject: [PATCH 01/25] add storage.ts, to replace KVStore storage.ts is replacing storage.js which had the KVStore service inside it. storage.ts will provide a set of functions performing the same duties as KVStorage's functions did. - `get` -> `storageGet` - `set` -> `storageSet` - `remove` -> `storageRemove` - `getDirect` -> `storageGetDirect` - `syncAllWebAndNativeValues` -> `storageSyncLocalAndNative` storage.ts will still use the "BEMUserCache" Cordova plugin in exactly the same way that KVStore did. However, instead of using the `angular-local-storage` package as a wrapper around localStorage, we will use it directly. A couple functions were added ('localStorageSet` and `localStorageGet`) to facilitate using localStorage directly - localStorage requires us to stringify objects before storing, and parse them on retrieval. Other than these substitutions, and being rewritten in modern JS with some typings, the logic is exactly the same as it was in KVStore. --To facilitate this change, storage.js is temporarily renamed to ngStorage.js so that it doesn't conflict with the storage.ts filename. ngStorage.js will be removed soon after it is not used anymore. --- www/index.js | 2 +- www/js/plugin/{storage.js => ngStorage.js} | 0 www/js/plugin/storage.ts | 182 +++++++++++++++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) rename www/js/plugin/{storage.js => ngStorage.js} (100%) create mode 100644 www/js/plugin/storage.ts diff --git a/www/index.js b/www/index.js index 55cb233b5..39304165c 100644 --- a/www/index.js +++ b/www/index.js @@ -31,4 +31,4 @@ import './js/control/uploadService.js'; import './js/metrics-factory.js'; import './js/metrics-mappings.js'; import './js/plugin/logger.ts'; -import './js/plugin/storage.js'; +import './js/plugin/ngStorage.js'; diff --git a/www/js/plugin/storage.js b/www/js/plugin/ngStorage.js similarity index 100% rename from www/js/plugin/storage.js rename to www/js/plugin/ngStorage.js diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts new file mode 100644 index 000000000..04ca3d539 --- /dev/null +++ b/www/js/plugin/storage.ts @@ -0,0 +1,182 @@ +import { getAngularService } from "../angular-react-helper"; +import { displayErrorMsg, logDebug } from "./logger"; + +const mungeValue = (key, value) => { + let store_val = value; + if (typeof value != "object") { + // Should this be {"value": value} or {key: value}? + store_val = {}; + store_val[key] = value; + } + return store_val; +} + +/* + * If a non-JSON object was munged for storage, unwrap it. + */ +const unmungeValue = (key, retData) => { + if (retData?.[key]) { + // it must have been a simple data type that we munged upfront + return retData[key]; + } else { + // it must have been an object + return retData; + } +} + +const localStorageSet = (key: string, value: {[k: string]: any}) => { + localStorage.setItem(key, JSON.stringify(value)); +} + +const localStorageGet = (key: string) => { + const value = localStorage.getItem(key); + if (value) { + return JSON.parse(value); + } else { + return null; + } +} + +/* We redundantly store data in both local and native storage. This function checks + both for a value. If a value is present in only one, it copies it to the other and returns it. + If a value is present in both, but they are different, it copies the native value to + local storage and returns it. */ +function getUnifiedValue(key) { + let ls_stored_val = localStorageGet(key); + return window['cordova'].plugins.BEMUserCache.getLocalStorage(key, false).then((uc_stored_val) => { + logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + ls_stored_val = ${JSON.stringify(ls_stored_val)}.`); + + /* compare stored values by stringified JSON equality, not by == or ===. + for objects, == or === only compares the references, not the contents of the objects */ + if (JSON.stringify(ls_stored_val) == JSON.stringify(uc_stored_val)) { + logDebug("local and native values match, already synced"); + return uc_stored_val; + } else { + // the values are different + if (ls_stored_val == null) { + console.assert(uc_stored_val != null, "uc_stored_val should be non-null"); + logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + ls_stored_val = ${JSON.stringify(ls_stored_val)}. + copying native ${key} to local...`); + localStorageSet(key, uc_stored_val); + return uc_stored_val; + } else if (uc_stored_val == null) { + console.assert(ls_stored_val != null); + /* + * Backwards compatibility ONLY. Right after the first + * update to this version, we may have a local value that + * is not a JSON object. In that case, we want to munge it + * before storage. Remove this after a few releases. + */ + ls_stored_val = mungeValue(key, ls_stored_val); + displayErrorMsg(`Local ${key} found, native ${key} missing, writing ${key} to native`); + logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + ls_stored_val = ${JSON.stringify(ls_stored_val)}. + copying local ${key} to native...`); + return window['cordova'].plugins.BEMUserCache.putLocalStorage(key, ls_stored_val).then(() => { + // we only return the value after we have finished writing + return ls_stored_val; + }); + } + console.assert(ls_stored_val != null && uc_stored_val != null, + "ls_stored_val =" + JSON.stringify(ls_stored_val) + + "uc_stored_val =" + JSON.stringify(uc_stored_val)); + displayErrorMsg(`Local ${key} found, native ${key} found, but different, + writing ${key} to local`); + logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + ls_stored_val = ${JSON.stringify(ls_stored_val)}. + copying native ${key} to local...`); + localStorageSet(key, uc_stored_val); + return uc_stored_val; + } + }); +} + +export function storageSet(key: string, value: any) { + const storeVal = mungeValue(key, value); + /* + * How should we deal with consistency here? Have the threads be + * independent so that there is greater chance that one will succeed, + * or the local only succeed if native succeeds. I think parallel is + * better for greater robustness. + */ + localStorageSet(key, storeVal); + return window['cordova'].plugins.BEMUserCache.putLocalStorage(key, storeVal); +} + +export function storageGet(key: string) { + return getUnifiedValue(key).then((retData) => unmungeValue(key, retData)); +} + +export function storageRemove(key: string) { + localStorage.removeItem(key); + return window['cordova'].plugins.BEMUserCache.removeLocalStorage(key); +} + +export function storageClear({ local, native }: { local?: boolean, native?: boolean }) { + if (local) localStorage.clear(); + if (native) return window['cordova'].plugins.BEMUserCache.clearAll(); + return Promise.resolve(); +} + +export function storageGetDirect(key: string) { + // will run in background, we won't wait for the results + getUnifiedValue(key); + return unmungeValue(key, localStorageGet(key)); +} + +function findMissing(fromKeys, toKeys) { + const foundKeys = []; + const missingKeys = []; + fromKeys.forEach((fk) => { + if (toKeys.includes(fk)) { + foundKeys.push(fk); + } else { + missingKeys.push(fk); + } + }); + return [foundKeys, missingKeys]; +} + +export function storageSyncLocalAndNative() { + const ClientStats = getAngularService('ClientStats'); + console.log("STORAGE_PLUGIN: Called syncAllWebAndNativeValues "); + const syncKeys = window['cordova'].plugins.BEMUserCache.listAllLocalStorageKeys().then((nativeKeys) => { + console.log("STORAGE_PLUGIN: native plugin returned"); + const webKeys = Object.keys(localStorage); + // I thought about iterating through the lists and copying over + // only missing values, etc but `getUnifiedValue` already does + // that, and we don't need to copy it + // so let's just find all the missing values and read them + logDebug("STORAGE_PLUGIN: Comparing web keys " + webKeys + " with " + nativeKeys); + let [foundNative, missingNative] = findMissing(webKeys, nativeKeys); + let [foundWeb, missingWeb] = findMissing(nativeKeys, webKeys); + logDebug("STORAGE_PLUGIN: Found native keys " + foundNative + " missing native keys " + missingNative); + logDebug("STORAGE_PLUGIN: Found web keys " + foundWeb + " missing web keys " + missingWeb); + const allMissing = missingNative.concat(missingWeb); + logDebug("STORAGE_PLUGIN: Syncing all missing keys " + allMissing); + allMissing.forEach(getUnifiedValue); + if (allMissing.length != 0) { + ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + "type": "local_storage_mismatch", + "allMissingLength": allMissing.length, + "missingWebLength": missingWeb.length, + "missingNativeLength": missingNative.length, + "foundWebLength": foundWeb.length, + "foundNativeLength": foundNative.length, + "allMissing": allMissing, + }).then(logDebug("Logged missing keys to client stats")); + } + }); + const listAllKeys = window['cordova'].plugins.BEMUserCache.listAllUniqueKeys().then((nativeKeys) => { + logDebug("STORAGE_PLUGIN: For the record, all unique native keys are " + nativeKeys); + if (nativeKeys.length == 0) { + ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + "type": "all_native", + }).then(logDebug("Logged all missing native keys to client stats")); + } + }); + + return Promise.all([syncKeys, listAllKeys]); +} From b99e0010c607372dcd9484b7d9015aa0b28088a7 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 27 Sep 2023 18:16:48 -0400 Subject: [PATCH 02/25] use storage.ts everywhere instead of KVStore storage.ts replaces KVStore, so we can replace all uses of KVStore throughout the codebase, substituting the new functions from storage.ts as follows: - `get` -> `storageGet` - `set` -> `storageSet` - `remove` -> `storageRemove` - `getDirect` -> `storageGetDirect` - `syncAllWebAndNativeValues` -> `storageSyncLocalAndNative` --- www/js/config/dynamicConfig.ts | 14 +++++--------- www/js/control/ProfileSettings.jsx | 8 ++++---- www/js/metrics-factory.js | 7 ++++--- www/js/metrics-mappings.js | 5 +++-- www/js/onboarding/SaveQrPage.tsx | 4 ++-- www/js/onboarding/onboardingHelper.ts | 7 +++---- www/js/splash/referral.js | 27 ++++++++++++++------------- www/js/splash/startprefs.js | 9 +++++---- 8 files changed, 40 insertions(+), 41 deletions(-) diff --git a/www/js/config/dynamicConfig.ts b/www/js/config/dynamicConfig.ts index f88255d4c..7994391a4 100644 --- a/www/js/config/dynamicConfig.ts +++ b/www/js/config/dynamicConfig.ts @@ -2,6 +2,7 @@ import i18next from "i18next"; import { displayError, logDebug, logWarn } from "../plugin/logger"; import { getAngularService } from "../angular-react-helper"; import { fetchUrlCached } from "../commHelper"; +import { storageClear, storageGet, storageSet } from "../plugin/storage"; export const CONFIG_PHONE_UI="config/app_ui_config"; export const CONFIG_PHONE_UI_KVSTORE ="CONFIG_PHONE_UI"; @@ -174,7 +175,6 @@ function extractSubgroup(token, config) { * @returns {boolean} boolean representing whether the config was updated or not */ function loadNewConfig(newToken, existingVersion = null) { - const KVStore = getAngularService('KVStore'); const newStudyLabel = extractStudyName(newToken); return readConfigFromServer(newStudyLabel).then((downloadedConfig) => { if (downloadedConfig.version == existingVersion) { @@ -190,7 +190,7 @@ function loadNewConfig(newToken, existingVersion = null) { } const storeConfigPromise = window['cordova'].plugins.BEMUserCache.putRWDocument( CONFIG_PHONE_UI, toSaveConfig); - const storeInKVStorePromise = KVStore.set(CONFIG_PHONE_UI_KVSTORE, toSaveConfig); + const storeInKVStorePromise = storageSet(CONFIG_PHONE_UI_KVSTORE, toSaveConfig); // loaded new config, so it is both ready and changed return Promise.all([storeConfigPromise, storeInKVStorePromise]).then( ([result, kvStoreResult]) => { @@ -220,17 +220,13 @@ export function initByUser(urlComponents) { } export function resetDataAndRefresh() { - const KVStore = getAngularService('KVStore'); - const resetNativePromise = window['cordova'].plugins.BEMUserCache.putRWDocument(CONFIG_PHONE_UI, {}); - const resetKVStorePromise = KVStore.clearAll(); - return Promise.all([resetNativePromise, resetKVStorePromise]) - .then(() => window.location.reload()); + // const resetNativePromise = window['cordova'].plugins.BEMUserCache.putRWDocument(CONFIG_PHONE_UI, {}); + storageClear({ local: true, native: true }).then(() => window.location.reload()); } export function getConfig() { if (storedConfig) return Promise.resolve(storedConfig); - const KVStore = getAngularService('KVStore'); - return KVStore.get(CONFIG_PHONE_UI_KVSTORE).then((config) => { + return storageGet(CONFIG_PHONE_UI_KVSTORE).then((config) => { if (config) { storedConfig = config; return config; diff --git a/www/js/control/ProfileSettings.jsx b/www/js/control/ProfileSettings.jsx index 8035c9462..566c100c1 100644 --- a/www/js/control/ProfileSettings.jsx +++ b/www/js/control/ProfileSettings.jsx @@ -22,6 +22,7 @@ import ControlCollectionHelper, {getHelperCollectionSettings, getState, isMedium import { resetDataAndRefresh } from "../config/dynamicConfig"; import { AppContext } from "../App"; import { shareQR } from "../components/QrCode"; +import { storageClear } from "../plugin/storage"; //any pure functions can go outside const ProfileSettings = () => { @@ -35,7 +36,6 @@ const ProfileSettings = () => { const CarbonDatasetHelper = getAngularService('CarbonDatasetHelper'); const UploadHelper = getAngularService('UploadHelper'); const EmailHelper = getAngularService('EmailHelper'); - const KVStore = getAngularService('KVStore'); const NotificationScheduler = getAngularService('NotificationScheduler'); const ControlHelper = getAngularService('ControlHelper'); const ClientStats = getAngularService('ClientStats'); @@ -387,15 +387,15 @@ const ProfileSettings = () => { style={settingStyles.dialog(colors.elevation.level3)}> {t('general-settings.clear-data')} - - - diff --git a/www/js/metrics-factory.js b/www/js/metrics-factory.js index ce813fbaa..4fc29fa7a 100644 --- a/www/js/metrics-factory.js +++ b/www/js/metrics-factory.js @@ -3,6 +3,7 @@ import angular from 'angular'; import { getBaseModeByValue } from './diary/diaryHelper' import { labelOptions } from './survey/multilabel/confirmHelper'; +import { storageGet, storageRemove, storageSet } from './plugin/storage'; angular.module('emission.main.metrics.factory', ['emission.main.metrics.mappings', @@ -164,13 +165,13 @@ angular.module('emission.main.metrics.factory', } cc.set = function(info) { - return KVStore.set(USER_DATA_KEY, info); + return storageSet(USER_DATA_KEY, info); }; cc.get = function() { - return KVStore.get(USER_DATA_KEY); + return storageGet(USER_DATA_KEY); }; cc.delete = function() { - return KVStore.remove(USER_DATA_KEY); + return storageRemove(USER_DATA_KEY); }; Number.prototype.between = function (min, max) { return this >= min && this <= max; diff --git a/www/js/metrics-mappings.js b/www/js/metrics-mappings.js index 2b71df739..91db216da 100644 --- a/www/js/metrics-mappings.js +++ b/www/js/metrics-mappings.js @@ -1,6 +1,7 @@ import angular from 'angular'; import { getLabelOptions } from './survey/multilabel/confirmHelper'; import { getConfig } from './config/dynamicConfig'; +import { storageGet, storageSet } from './plugin/storage'; angular.module('emission.main.metrics.mappings', ['emission.plugin.logger', 'emission.plugin.kvstore']) @@ -146,7 +147,7 @@ angular.module('emission.main.metrics.mappings', ['emission.plugin.logger', } this.loadCarbonDatasetLocale = function() { - return KVStore.get(CARBON_DATASET_KEY).then(function(localeCode) { + return storageGet(CARBON_DATASET_KEY).then(function(localeCode) { Logger.log("CarbonDatasetHelper.loadCarbonDatasetLocale() obtained value from storage [" + localeCode + "]"); if (!localeCode) { localeCode = defaultCarbonDatasetCode; @@ -158,7 +159,7 @@ angular.module('emission.main.metrics.mappings', ['emission.plugin.logger', this.saveCurrentCarbonDatasetLocale = function (localeCode) { setCurrentCarbonDatasetLocale(localeCode); - KVStore.set(CARBON_DATASET_KEY, currentCarbonDatasetCode); + storageSet(CARBON_DATASET_KEY, currentCarbonDatasetCode); Logger.log("CarbonDatasetHelper.saveCurrentCarbonDatasetLocale() saved value [" + currentCarbonDatasetCode + "] to storage"); } diff --git a/www/js/onboarding/SaveQrPage.tsx b/www/js/onboarding/SaveQrPage.tsx index 157ff4093..57fe8f679 100644 --- a/www/js/onboarding/SaveQrPage.tsx +++ b/www/js/onboarding/SaveQrPage.tsx @@ -10,6 +10,7 @@ import { useTranslation } from "react-i18next"; import QrCode, { shareQR } from "../components/QrCode"; import { onboardingStyles } from "./OnboardingStack"; import { preloadDemoSurveyResponse } from "./SurveyPage"; +import { storageSet } from "../plugin/storage"; const SaveQrPage = ({ }) => { @@ -33,10 +34,9 @@ const SaveQrPage = ({ }) => { function login(token) { const CommHelper = getAngularService('CommHelper'); - const KVStore = getAngularService('KVStore'); const EXPECTED_METHOD = "prompted-auth"; const dbStorageObject = {"token": token}; - return KVStore.set(EXPECTED_METHOD, dbStorageObject).then((r) => { + return storageSet(EXPECTED_METHOD, dbStorageObject).then((r) => { CommHelper.registerUser((successResult) => { refreshOnboardingState(); }, function(errorResult) { diff --git a/www/js/onboarding/onboardingHelper.ts b/www/js/onboarding/onboardingHelper.ts index 659382079..b1cdff0ed 100644 --- a/www/js/onboarding/onboardingHelper.ts +++ b/www/js/onboarding/onboardingHelper.ts @@ -1,6 +1,7 @@ import { DateTime } from "luxon"; import { getAngularService } from "../angular-react-helper"; import { getConfig } from "../config/dynamicConfig"; +import { storageGet, storageSet } from "../plugin/storage"; export const INTRO_DONE_KEY = 'intro_done'; @@ -39,12 +40,10 @@ async function readConsented() { } async function readIntroDone() { - const KVStore = getAngularService('KVStore'); - return KVStore.get(INTRO_DONE_KEY).then((read_val) => !!read_val) as Promise; + return storageGet(INTRO_DONE_KEY).then((read_val) => !!read_val) as Promise; } export async function markIntroDone() { const currDateTime = DateTime.now().toISO(); - const KVStore = getAngularService('KVStore'); - return KVStore.set(INTRO_DONE_KEY, currDateTime); + return storageSet(INTRO_DONE_KEY, currDateTime); } diff --git a/www/js/splash/referral.js b/www/js/splash/referral.js index 9e4707200..4c0d2823b 100644 --- a/www/js/splash/referral.js +++ b/www/js/splash/referral.js @@ -1,4 +1,5 @@ import angular from 'angular'; +import { storageGetDirect, storageRemove, storageSet } from '../plugin/storage'; angular.module('emission.splash.referral', ['emission.plugin.kvstore']) @@ -11,32 +12,32 @@ angular.module('emission.splash.referral', ['emission.plugin.kvstore']) var REFERRED_USER_ID = 'referred_user_id'; referralHandler.getReferralNavigation = function() { - const toReturn = KVStore.getDirect(REFERRAL_NAVIGATION_KEY); - KVStore.remove(REFERRAL_NAVIGATION_KEY); + const toReturn = storageGetDirect(REFERRAL_NAVIGATION_KEY); + storageRemove(REFERRAL_NAVIGATION_KEY); return toReturn; } referralHandler.setupGroupReferral = function(kvList) { - KVStore.set(REFERRED_KEY, true); - KVStore.set(REFERRED_GROUP_ID, kvList['groupid']); - KVStore.set(REFERRED_USER_ID, kvList['userid']); - KVStore.set(REFERRAL_NAVIGATION_KEY, 'goals'); + storageSet(REFERRED_KEY, true); + storageSet(REFERRED_GROUP_ID, kvList['groupid']); + storageSet(REFERRED_USER_ID, kvList['userid']); + storageSet(REFERRAL_NAVIGATION_KEY, 'goals'); }; referralHandler.clearGroupReferral = function(kvList) { - KVStore.remove(REFERRED_KEY); - KVStore.remove(REFERRED_GROUP_ID); - KVStore.remove(REFERRED_USER_ID); - KVStore.remove(REFERRAL_NAVIGATION_KEY); + storageRemove(REFERRED_KEY); + storageRemove(REFERRED_GROUP_ID); + storageRemove(REFERRED_USER_ID); + storageRemove(REFERRAL_NAVIGATION_KEY); }; referralHandler.getReferralParams = function(kvList) { - return [KVStore.getDirect(REFERRED_GROUP_ID), - KVStore.getDirect(REFERRED_USER_ID)]; + return [storageGetDirect(REFERRED_GROUP_ID), + storageGetDirect(REFERRED_USER_ID)]; } referralHandler.hasPendingRegistration = function() { - return KVStore.getDirect(REFERRED_KEY) + return storageGetDirect(REFERRED_KEY) }; return referralHandler; diff --git a/www/js/splash/startprefs.js b/www/js/splash/startprefs.js index e535d179a..bec593131 100644 --- a/www/js/splash/startprefs.js +++ b/www/js/splash/startprefs.js @@ -1,5 +1,6 @@ import angular from 'angular'; import { getConfig } from '../config/dynamicConfig'; +import { storageGet, storageSet } from '../plugin/storage'; angular.module('emission.splash.startprefs', ['emission.plugin.logger', 'emission.splash.referral', @@ -31,7 +32,7 @@ angular.module('emission.splash.startprefs', ['emission.plugin.logger', // mark in native storage return startprefs.readConsentState().then(writeConsentToNative).then(function(response) { // mark in local storage - KVStore.set(DATA_COLLECTION_CONSENTED_PROTOCOL, + storageSet(DATA_COLLECTION_CONSENTED_PROTOCOL, $rootScope.req_consent); // mark in local variable as well $rootScope.curr_consented = angular.copy($rootScope.req_consent); @@ -41,13 +42,13 @@ angular.module('emission.splash.startprefs', ['emission.plugin.logger', startprefs.markIntroDone = function() { var currTime = moment().format(); - KVStore.set(INTRO_DONE_KEY, currTime); + storageSet(INTRO_DONE_KEY, currTime); $rootScope.$emit(startprefs.INTRO_DONE_EVENT, currTime); } // returns boolean startprefs.readIntroDone = function() { - return KVStore.get(INTRO_DONE_KEY).then(function(read_val) { + return storageGet(INTRO_DONE_KEY).then(function(read_val) { logger.log("in readIntroDone, read_val = "+JSON.stringify(read_val)); $rootScope.intro_done = read_val; }); @@ -84,7 +85,7 @@ angular.module('emission.splash.startprefs', ['emission.plugin.logger', .then(function(startupConfigResult) { $rootScope.req_consent = startupConfigResult.data.emSensorDataCollectionProtocol; logger.log("required consent version = " + JSON.stringify($rootScope.req_consent)); - return KVStore.get(DATA_COLLECTION_CONSENTED_PROTOCOL); + return storageGet(DATA_COLLECTION_CONSENTED_PROTOCOL); }).then(function(kv_store_consent) { $rootScope.curr_consented = kv_store_consent; console.assert(angular.isDefined($rootScope.req_consent), "in readConsentState $rootScope.req_consent", JSON.stringify($rootScope.req_consent)); From 6ffbf1886218898dba60e7294e51158084152f1a Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 27 Sep 2023 18:21:39 -0400 Subject: [PATCH 03/25] remove KVStore / ngStorage.js Rewritten into storage.ts, this Angular service is not needed anymore. We can remove the file, remove it from index.js, and remove it as a dependency from all of the Angular modules that previously used it. --- www/index.js | 1 - www/js/metrics-factory.js | 5 +- www/js/metrics-mappings.js | 7 +- www/js/plugin/ngStorage.js | 221 ------------------------------------ www/js/services.js | 3 +- www/js/splash/referral.js | 4 +- www/js/splash/startprefs.js | 5 +- 7 files changed, 10 insertions(+), 236 deletions(-) delete mode 100644 www/js/plugin/ngStorage.js diff --git a/www/index.js b/www/index.js index 39304165c..a858ef784 100644 --- a/www/index.js +++ b/www/index.js @@ -31,4 +31,3 @@ import './js/control/uploadService.js'; import './js/metrics-factory.js'; import './js/metrics-mappings.js'; import './js/plugin/logger.ts'; -import './js/plugin/ngStorage.js'; diff --git a/www/js/metrics-factory.js b/www/js/metrics-factory.js index 4fc29fa7a..b5ead9c82 100644 --- a/www/js/metrics-factory.js +++ b/www/js/metrics-factory.js @@ -6,8 +6,7 @@ import { labelOptions } from './survey/multilabel/confirmHelper'; import { storageGet, storageRemove, storageSet } from './plugin/storage'; angular.module('emission.main.metrics.factory', - ['emission.main.metrics.mappings', - 'emission.plugin.kvstore']) + ['emission.main.metrics.mappings']) .factory('FootprintHelper', function(CarbonDatasetHelper, CustomDatasetHelper) { var fh = {}; @@ -145,7 +144,7 @@ angular.module('emission.main.metrics.factory', return fh; }) -.factory('CalorieCal', function(KVStore, METDatasetHelper, CustomDatasetHelper) { +.factory('CalorieCal', function(METDatasetHelper, CustomDatasetHelper) { var cc = {}; var highestMET = 0; diff --git a/www/js/metrics-mappings.js b/www/js/metrics-mappings.js index 91db216da..60068711d 100644 --- a/www/js/metrics-mappings.js +++ b/www/js/metrics-mappings.js @@ -3,10 +3,9 @@ import { getLabelOptions } from './survey/multilabel/confirmHelper'; import { getConfig } from './config/dynamicConfig'; import { storageGet, storageSet } from './plugin/storage'; -angular.module('emission.main.metrics.mappings', ['emission.plugin.logger', - 'emission.plugin.kvstore']) +angular.module('emission.main.metrics.mappings', ['emission.plugin.logger']) -.service('CarbonDatasetHelper', function(KVStore) { +.service('CarbonDatasetHelper', function() { var CARBON_DATASET_KEY = 'carbon_dataset_locale'; // Values are in Kg/PKm (kilograms per passenger-kilometer) @@ -182,7 +181,7 @@ angular.module('emission.main.metrics.mappings', ['emission.plugin.logger', return carbonDatasets[currentCarbonDatasetCode].footprintData; }; }) -.service('METDatasetHelper', function(KVStore) { +.service('METDatasetHelper', function() { var standardMETs = { "WALKING": { "VERY_SLOW": { diff --git a/www/js/plugin/ngStorage.js b/www/js/plugin/ngStorage.js deleted file mode 100644 index a14b1db83..000000000 --- a/www/js/plugin/ngStorage.js +++ /dev/null @@ -1,221 +0,0 @@ -import angular from 'angular'; - -angular.module('emission.plugin.kvstore', ['emission.plugin.logger', - 'LocalStorageModule', - 'emission.stats.clientstats']) - -.factory('KVStore', function($window, Logger, localStorageService, $ionicPopup, - $ionicPlatform, ClientStats) { - var logger = Logger; - var kvstoreJs = {} - /* - * Sets in both localstorage and native storage - * If the message is not a JSON object, wrap it in an object with the key - * "value" before storing it. - */ - var getNativePlugin = function() { - return $window.cordova.plugins.BEMUserCache; - } - - /* - * Munge plain, non-JSON objects to JSON objects before storage - */ - - var mungeValue = function(key, value) { - var store_val = value; - if (typeof value != "object") { - // Should this be {"value": value} or {key: value}? - store_val = {}; - store_val[key] = value; - } - return store_val; - } - - - kvstoreJs.set = function(key, value) { - // add checks for data type - var store_val = mungeValue(key, value); - /* - * How should we deal with consistency here? Have the threads be - * independent so that there is greater chance that one will succeed, - * or the local only succeed if native succeeds. I think parallel is - * better for greater robustness. - */ - localStorageService.set(key, store_val); - return getNativePlugin().putLocalStorage(key, store_val); - } - - var getUnifiedValue = function(key) { - var ls_stored_val = localStorageService.get(key, undefined); - return getNativePlugin().getLocalStorage(key, false).then(function(uc_stored_val) { - logger.log("for key "+key+" uc_stored_val = "+JSON.stringify(uc_stored_val)+" ls_stored_val = "+JSON.stringify(ls_stored_val)); - if (angular.equals(ls_stored_val, uc_stored_val)) { - logger.log("local and native values match, already synced"); - return uc_stored_val; - } else { - // the values are different - if (ls_stored_val == null) { - console.assert(uc_stored_val != null, "uc_stored_val should be non-null"); - logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ - " ls_stored_val = "+JSON.stringify(ls_stored_val)+ - " copying native "+key+" to local..."); - localStorageService.set(key, uc_stored_val); - return uc_stored_val; - } else if (uc_stored_val == null) { - console.assert(ls_stored_val != null); - /* - * Backwards compatibility ONLY. Right after the first - * update to this version, we may have a local value that - * is not a JSON object. In that case, we want to munge it - * before storage. Remove this after a few releases. - */ - ls_stored_val = mungeValue(key, ls_stored_val); - $ionicPopup.alert({template: "Local "+key+" found, native " - +key+" missing, writing "+key+" to native"}) - logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ - " ls_stored_val = "+JSON.stringify(ls_stored_val)+ - " copying local "+key+" to native..."); - return getNativePlugin().putLocalStorage(key, ls_stored_val).then(function() { - // we only return the value after we have finished writing - return ls_stored_val; - }); - } - console.assert(ls_stored_val != null && uc_stored_val != null, - "ls_stored_val ="+JSON.stringify(ls_stored_val)+ - "uc_stored_val ="+JSON.stringify(uc_stored_val)); - $ionicPopup.alert({template: "Local "+key+" found, native " - +key+" found, but different, writing "+key+" to local"}) - logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ - " ls_stored_val = "+JSON.stringify(ls_stored_val)+ - " copying native "+key+" to local..."); - localStorageService.set(key, uc_stored_val); - return uc_stored_val; - } - }); - } - - /* - * If a non-JSON object was munged for storage, unwrap it. - */ - var unmungeValue = function(key, retData) { - if((retData != null) && (angular.isDefined(retData[key]))) { - // it must have been a simple data type that we munged upfront - return retData[key]; - } else { - // it must have been an object - return retData; - } - } - - kvstoreJs.get = function(key) { - return getUnifiedValue(key).then(function(retData) { - return unmungeValue(key, retData); - }); - } - - /* - * TODO: Temporary fix for data that: - - we want to return inline instead of in a promise - - is not catastrophic if it is cleared out (e.g. walkthrough code), OR - - is used primarily for session storage so will not be cleared out - (e.g. referral code) - We can replace this with promises in a future PR if needed - - The code does copy the native value to local storage in the background, - so even if this is stripped out, it will work on retry. - */ - kvstoreJs.getDirect = function(key) { - // will run in background, we won't wait for the results - getUnifiedValue(key); - return unmungeValue(key, localStorageService.get(key)); - } - - kvstoreJs.remove = function(key) { - localStorageService.remove(key); - return getNativePlugin().removeLocalStorage(key); - } - - kvstoreJs.clearAll = function() { - localStorageService.clearAll(); - return getNativePlugin().clearAll(); - } - - /* - * Unfortunately, there is weird deletion of native - * https://github.com/e-mission/e-mission-docs/issues/930 - * So we cannot remove this if/until we switch to react native - */ - kvstoreJs.clearOnlyLocal = function() { - return localStorageService.clearAll(); - } - - kvstoreJs.clearOnlyNative = function() { - return getNativePlugin().clearAll(); - } - - let findMissing = function(fromKeys, toKeys) { - const foundKeys = []; - const missingKeys = []; - fromKeys.forEach((fk) => { - if (toKeys.includes(fk)) { - foundKeys.push(fk); - } else { - missingKeys.push(fk); - } - }); - return [foundKeys, missingKeys]; - } - - let syncAllWebAndNativeValues = function() { - console.log("STORAGE_PLUGIN: Called syncAllWebAndNativeValues "); - const syncKeys = getNativePlugin().listAllLocalStorageKeys().then((nativeKeys) => { - console.log("STORAGE_PLUGIN: native plugin returned"); - const webKeys = localStorageService.keys(); - // I thought about iterating through the lists and copying over - // only missing values, etc but `getUnifiedValue` already does - // that, and we don't need to copy it - // so let's just find all the missing values and read them - logger.log("STORAGE_PLUGIN: Comparing web keys "+webKeys+" with "+nativeKeys); - let [foundNative, missingNative] = findMissing(webKeys, nativeKeys); - let [foundWeb, missingWeb] = findMissing(nativeKeys, webKeys); - logger.log("STORAGE_PLUGIN: Found native keys "+foundNative+" missing native keys "+missingNative); - logger.log("STORAGE_PLUGIN: Found web keys "+foundWeb+" missing web keys "+missingWeb); - const allMissing = missingNative.concat(missingWeb); - logger.log("STORAGE_PLUGIN: Syncing all missing keys "+allMissing); - allMissing.forEach(getUnifiedValue); - if (allMissing.length != 0) { - ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { - "type": "local_storage_mismatch", - "allMissingLength": allMissing.length, - "missingWebLength": missingWeb.length, - "missingNativeLength": missingNative.length, - "foundWebLength": foundWeb.length, - "foundNativeLength": foundNative.length, - "allMissing": allMissing, - }).then(Logger.log("Logged missing keys to client stats")); - } - }); - const listAllKeys = getNativePlugin().listAllUniqueKeys().then((nativeKeys) => { - logger.log("STORAGE_PLUGIN: For the record, all unique native keys are "+nativeKeys); - if (nativeKeys.length == 0) { - ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { - "type": "all_native", - }).then(Logger.log("Logged all missing native keys to client stats")); - } - }); - - return Promise.all([syncKeys, listAllKeys]); - } - - $ionicPlatform.ready().then(function() { - Logger.log("STORAGE_PLUGIN: app launched, checking storage sync"); - syncAllWebAndNativeValues(); - }); - - $ionicPlatform.on("resume", function() { - Logger.log("STORAGE_PLUGIN: app has resumed, checking storage sync"); - syncAllWebAndNativeValues(); - }); - - return kvstoreJs; -}); diff --git a/www/js/services.js b/www/js/services.js index 9a63b364d..1955cd4bb 100644 --- a/www/js/services.js +++ b/www/js/services.js @@ -2,8 +2,7 @@ import angular from 'angular'; -angular.module('emission.services', ['emission.plugin.logger', - 'emission.plugin.kvstore']) +angular.module('emission.services', ['emission.plugin.logger']) .service('CommHelper', function($rootScope) { var getConnectURL = function(successCallback, errorCallback) { diff --git a/www/js/splash/referral.js b/www/js/splash/referral.js index 4c0d2823b..849de847a 100644 --- a/www/js/splash/referral.js +++ b/www/js/splash/referral.js @@ -1,9 +1,9 @@ import angular from 'angular'; import { storageGetDirect, storageRemove, storageSet } from '../plugin/storage'; -angular.module('emission.splash.referral', ['emission.plugin.kvstore']) +angular.module('emission.splash.referral', []) -.factory('ReferralHandler', function($window, KVStore) { +.factory('ReferralHandler', function($window) { var referralHandler = {}; var REFERRAL_NAVIGATION_KEY = 'referral_navigation'; diff --git a/www/js/splash/startprefs.js b/www/js/splash/startprefs.js index bec593131..92a07e624 100644 --- a/www/js/splash/startprefs.js +++ b/www/js/splash/startprefs.js @@ -3,11 +3,10 @@ import { getConfig } from '../config/dynamicConfig'; import { storageGet, storageSet } from '../plugin/storage'; angular.module('emission.splash.startprefs', ['emission.plugin.logger', - 'emission.splash.referral', - 'emission.plugin.kvstore']) + 'emission.splash.referral']) .factory('StartPrefs', function($window, $state, $interval, $rootScope, $ionicPlatform, - $ionicPopup, KVStore, $http, Logger, ReferralHandler) { + $ionicPopup, $http, Logger, ReferralHandler) { var logger = Logger; var startprefs = {}; // Boolean: represents that the "intro" - the one page summary From cdabde43a743f482325bb021d8cce284831aeebc Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 27 Sep 2023 20:54:36 -0400 Subject: [PATCH 04/25] use JSDOM as jest env so localStorage works Followed the advice of https://stackoverflow.com/a/74309873 so that we can have a mocked out localStorage to use in tests --- jest.config.json | 1 + package.serve.json | 1 + www/__tests__/storage.test.ts | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 www/__tests__/storage.test.ts diff --git a/jest.config.json b/jest.config.json index 78dc839b4..77b53782c 100644 --- a/jest.config.json +++ b/jest.config.json @@ -1,4 +1,5 @@ { + "testEnvironment": "jsdom", "testPathIgnorePatterns": [ "/node_modules/", "/platforms/", diff --git a/package.serve.json b/package.serve.json index 1a2ef6cb0..57470bc2d 100644 --- a/package.serve.json +++ b/package.serve.json @@ -35,6 +35,7 @@ "expose-loader": "^4.1.0", "file-loader": "^6.2.0", "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", "phonegap": "9.0.0+cordova.9.0.0", "process": "^0.11.10", "sass": "^1.62.1", diff --git a/www/__tests__/storage.test.ts b/www/__tests__/storage.test.ts new file mode 100644 index 000000000..841bd7ee3 --- /dev/null +++ b/www/__tests__/storage.test.ts @@ -0,0 +1,4 @@ +it('stores a value in localstorage and then retrieves it', () => { + localStorage.setItem("testKey", "testValue"); + expect(localStorage.getItem("testKey")).toBe("testValue"); +}); From fd4f9d54a66d1e05d89f15fdedce4e44e7c36ffb Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 00:51:06 -0400 Subject: [PATCH 05/25] add tests for storage.ts, plus the needed mocks To test storage.ts we need to be able to mock the usercache cordova plugin, as well as fill in the missing window.Logger. I created mocks for both of these which are kept in the __mocks__ folder and simply called at the start of storage.test.ts. Because storage.ts calls getAngularService, angular-react-helper.tsx was included in the process. That file imported PaperProvider, which did not work with jest. Because we are not angularizing components anymore, we do not actually need much of the code in angular-react-helper anyway. Let's just take it out so we can safely implicate getAngularService. As for the storage tests themselves, they check whether the storage functions are able to store and retrieve data, and crucially they ensure that local and native storage can compensate for each other if one of them gets cleared. --- jest.config.json | 3 ++ www/__mocks__/cordovaMocks.ts | 53 +++++++++++++++++++++++ www/__mocks__/globalMocks.ts | 3 ++ www/__tests__/storage.test.ts | 76 +++++++++++++++++++++++++++++++-- www/js/angular-react-helper.tsx | 55 ------------------------ 5 files changed, 132 insertions(+), 58 deletions(-) create mode 100644 www/__mocks__/cordovaMocks.ts create mode 100644 www/__mocks__/globalMocks.ts diff --git a/jest.config.json b/jest.config.json index 77b53782c..71bc5f5ca 100644 --- a/jest.config.json +++ b/jest.config.json @@ -10,6 +10,9 @@ "transform": { "^.+\\.(ts|tsx|js|jsx)$": "ts-jest" }, + "transformIgnorePatterns": [ + "/node_modules/(?!(@react-native|react-native|react-native-vector-icons))" + ], "moduleNameMapper": { "^react-native$": "react-native-web" } diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts new file mode 100644 index 000000000..5f723a5d7 --- /dev/null +++ b/www/__mocks__/cordovaMocks.ts @@ -0,0 +1,53 @@ +export const mockBEMUserCache = () => { + const _cache = {}; + const mockBEMUserCache = { + getLocalStorage: (key: string, isSecure: boolean) => { + return new Promise((rs, rj) => + setTimeout(() => { + rs(_cache[key]); + }, 100) + ); + }, + putLocalStorage: (key: string, value: any) => { + return new Promise((rs, rj) => + setTimeout(() => { + _cache[key] = value; + rs(); + }, 100) + ); + }, + removeLocalStorage: (key: string) => { + return new Promise((rs, rj) => + setTimeout(() => { + delete _cache[key]; + rs(); + }, 100) + ); + }, + clearAll: () => { + return new Promise((rs, rj) => + setTimeout(() => { + for (let p in _cache) delete _cache[p]; + rs(); + }, 100) + ); + }, + listAllLocalStorageKeys: () => { + return new Promise((rs, rj) => + setTimeout(() => { + rs(Object.keys(_cache)); + }, 100) + ); + }, + listAllUniqueKeys: () => { + return new Promise((rs, rj) => + setTimeout(() => { + rs(Object.keys(_cache)); + }, 100) + ); + } + } + window['cordova'] ||= {}; + window['cordova'].plugins ||= {}; + window['cordova'].plugins.BEMUserCache = mockBEMUserCache; +} diff --git a/www/__mocks__/globalMocks.ts b/www/__mocks__/globalMocks.ts new file mode 100644 index 000000000..3d9b71507 --- /dev/null +++ b/www/__mocks__/globalMocks.ts @@ -0,0 +1,3 @@ +export const mockLogger = () => { + window['Logger'] = { log: console.log }; +} diff --git a/www/__tests__/storage.test.ts b/www/__tests__/storage.test.ts index 841bd7ee3..6fea4f8b9 100644 --- a/www/__tests__/storage.test.ts +++ b/www/__tests__/storage.test.ts @@ -1,4 +1,74 @@ -it('stores a value in localstorage and then retrieves it', () => { - localStorage.setItem("testKey", "testValue"); - expect(localStorage.getItem("testKey")).toBe("testValue"); +import { mockBEMUserCache } from "../__mocks__/cordovaMocks"; +import { mockLogger } from "../__mocks__/globalMocks"; +import { storageClear, storageGet, storageRemove, storageSet } from "../js/plugin/storage"; + +// mocks used - storage.ts uses BEMUserCache and logging. +// localStorage is already mocked for us by Jest :) +mockLogger(); +mockBEMUserCache(); + +it('stores a value and retrieves it back', async () => { + await storageSet('test1', 'test value 1'); + const retVal = await storageGet('test1'); + expect(retVal).toEqual('test value 1'); +}); + +it('stores a value, removes it, and checks that it is gone', async () => { + await storageSet('test2', 'test value 2'); + await storageRemove('test2'); + const retVal = await storageGet('test2'); + expect(retVal).toBeUndefined(); +}); + +it('can store objects too', async () => { + const obj = { a: 1, b: 2 }; + await storageSet('test6', obj); + const retVal = await storageGet('test6'); + expect(retVal).toEqual(obj); +}); + +it('can also store complex nested objects with arrays', async () => { + const obj = { a: 1, b: { c: [1, 2, 3] } }; + await storageSet('test7', obj); + const retVal = await storageGet('test7'); + expect(retVal).toEqual(obj); +}); + +it('preserves values if local gets cleared', async () => { + await storageSet('test3', 'test value 3'); + await storageClear({ local: true }); + const retVal = await storageGet('test3'); + expect(retVal).toEqual('test value 3'); +}); + +it('preserves values if native gets cleared', async () => { + await storageSet('test4', 'test value 4'); + await storageClear({ native: true }); + const retVal = await storageGet('test4'); + expect(retVal).toEqual('test value 4'); +}); + +it('does not preserve values if both local and native are cleared', async () => { + await storageSet('test5', 'test value 5'); + await storageClear({ local: true, native: true }); + const retVal = await storageGet('test5'); + expect(retVal).toBeUndefined(); +}); + +it('preserves values if local gets cleared, then retrieved, then native gets cleared', async () => { + await storageSet('test8', 'test value 8'); + await storageClear({ local: true }); + await storageGet('test8'); + await storageClear({ native: true }); + const retVal = await storageGet('test8'); + expect(retVal).toEqual('test value 8'); +}); + +it('preserves values if native gets cleared, then retrieved, then local gets cleared', async () => { + await storageSet('test9', 'test value 9'); + await storageClear({ native: true }); + await storageGet('test9'); + await storageClear({ local: true }); + const retVal = await storageGet('test9'); + expect(retVal).toEqual('test value 9'); }); diff --git a/www/js/angular-react-helper.tsx b/www/js/angular-react-helper.tsx index 984e529ff..4813ae2e9 100644 --- a/www/js/angular-react-helper.tsx +++ b/www/js/angular-react-helper.tsx @@ -3,61 +3,6 @@ // Modified to use React 18 and wrap elements with the React Native Paper Provider import angular from 'angular'; -import { createRoot } from 'react-dom/client'; -import React from 'react'; -import { Provider as PaperProvider, MD3LightTheme as DefaultTheme, MD3Colors } from 'react-native-paper'; -import { getTheme } from './appTheme'; - -function toBindings(propTypes) { - const bindings = {}; - Object.keys(propTypes).forEach(key => bindings[key] = '<'); - return bindings; -} - -function toProps(propTypes, controller) { - const props = {}; - Object.keys(propTypes).forEach(key => props[key] = controller[key]); - return props; -} - -export function angularize(component, name, modulePath) { - component.module = modulePath; - const nameCamelCase = name[0].toLowerCase() + name.slice(1); - angular - .module(modulePath, []) - .component(nameCamelCase, makeComponentProps(component)); -} - -const theme = getTheme(); -export function makeComponentProps(Component) { - const propTypes = Component.propTypes || {}; - return { - bindings: toBindings(propTypes), - controller: ['$element', function($element) { - /* TODO: once the inf scroll list is converted to React and no longer uses - collection-repeat, we can just set the root here one time - and will not have to reassign it in $onChanges. */ - /* Until then, React will complain everytime we reassign an element's root */ - let root; - this.$onChanges = () => { - root = createRoot($element[0]); - const props = toProps(propTypes, this); - root.render( - - - - - ); - }; - this.$onDestroy = () => root.unmount(); - }] - }; -} export function getAngularService(name: string) { const injector = angular.element(document.body).injector(); From 8ada4f8b43d1751774cb4b446ded869e9590f419 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 01:15:17 -0400 Subject: [PATCH 06/25] add clientStats.js, to replace ClientStats service js/plugin/clientStats.ts will replace js/stats/clientstats.js - camelcase filename convention - we don't need a dedicated folder for stats because clientstats.js was the only thing in it Nothing significant has changed here. I just refactored into modern JS and added type definitions for each function's parameters. mappings of old/new exported methods: - getStatKeys -> statKeys (now a variable, not a function) - getAppVersion -> getAppVersion - getStatsEvent -> getStatsEvent - addReading -> addStatReading - addEvent -> addStatEvent - addError -> addStatError --- www/js/plugin/clientStats.ts | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 www/js/plugin/clientStats.ts diff --git a/www/js/plugin/clientStats.ts b/www/js/plugin/clientStats.ts new file mode 100644 index 000000000..b945ee5f8 --- /dev/null +++ b/www/js/plugin/clientStats.ts @@ -0,0 +1,48 @@ +const CLIENT_TIME = "stats/client_time"; +const CLIENT_ERROR = "stats/client_error"; +const CLIENT_NAV_EVENT = "stats/client_nav_event"; + +export const statKeys = { + STATE_CHANGED: "state_changed", + BUTTON_FORCE_SYNC: "button_sync_forced", + CHECKED_DIARY: "checked_diary", + DIARY_TIME: "diary_time", + METRICS_TIME: "metrics_time", + CHECKED_INF_SCROLL: "checked_inf_scroll", + INF_SCROLL_TIME: "inf_scroll_time", + VERIFY_TRIP: "verify_trip", + LABEL_TAB_SWITCH: "label_tab_switch", + SELECT_LABEL: "select_label", + EXPANDED_TRIP: "expanded_trip", + NOTIFICATION_OPEN: "notification_open", + REMINDER_PREFS: "reminder_time_prefs", + MISSING_KEYS: "missing_keys" +}; + +let appVersion; +export const getAppVersion = () => { + appVersion ||= window['cordova']?.plugins?.BEMUserCache?.getAppVersion(); + return appVersion; +} + +export const getStatsEvent = (name: string, reading: any) => { + const ts = Date.now() / 1000; + const client_app_version = getAppVersion(); + const client_os_version = window['device'].version; + return { name, ts, reading, client_app_version, client_os_version }; +} + +export const addStatReading = (name: string, reading: any) => { + const db = window['cordova']?.plugins?.BEMUserCache; + if (db) return db.putMessage(CLIENT_TIME, getStatsEvent(name, reading)); +} + +export const addStatEvent = (name: string) => { + const db = window['cordova']?.plugins?.BEMUserCache; + if (db) return db.putMessage(CLIENT_NAV_EVENT, getStatsEvent(name, null)); +} + +export const addStatError = (name: string, errorStr: string) => { + const db = window['cordova']?.plugins?.BEMUserCache; + if (db) return db.putMessage(CLIENT_ERROR, getStatsEvent(name, errorStr)); +} From 6e59af538d43e17d2a1cf230112c8134866ccc3f Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 01:29:51 -0400 Subject: [PATCH 07/25] use clientStats.ts everywhere, not old ClientStats ClientStats was rewritten into clientStats.ts. This commit simply substitutes in the new methods for the old ones, everywhere that client stats are recorded, as follows: - getStatKeys -> statKeys (now a variable, not a function) - getAppVersion -> getAppVersion - getStatsEvent -> getStatsEvent - addReading -> addStatReading - addEvent -> addStatEvent --- www/js/control/ControlSyncHelper.tsx | 7 ++++--- www/js/control/ProfileSettings.jsx | 3 ++- www/js/controllers.js | 10 ++++------ www/js/plugin/storage.ts | 5 +++-- www/js/splash/notifScheduler.js | 3 ++- www/js/splash/remotenotify.js | 8 ++++---- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/www/js/control/ControlSyncHelper.tsx b/www/js/control/ControlSyncHelper.tsx index 490672c4d..992d48562 100644 --- a/www/js/control/ControlSyncHelper.tsx +++ b/www/js/control/ControlSyncHelper.tsx @@ -8,6 +8,7 @@ import ActionMenu from "../components/ActionMenu"; import SettingRow from "./SettingRow"; import AlertBar from "./AlertBar"; import moment from "moment"; +import { addStatEvent, statKeys } from "../plugin/clientStats"; /* * BEGIN: Simple read/write wrappers @@ -61,8 +62,8 @@ export const ForceSyncRow = ({getState}) => { async function forceSync() { try { - let addedEvent = ClientStats.addEvent(ClientStats.getStatKeys().BUTTON_FORCE_SYNC); - console.log("Added "+ClientStats.getStatKeys().BUTTON_FORCE_SYNC+" event"); + let addedEvent = addStatEvent(statKeys.BUTTON_FORCE_SYNC); + console.log("Added "+statKeys.BUTTON_FORCE_SYNC+" event"); let sync = await forcePluginSync(); /* @@ -281,4 +282,4 @@ const ControlSyncHelper = ({ editVis, setEditVis }) => { ); }; -export default ControlSyncHelper; \ No newline at end of file +export default ControlSyncHelper; diff --git a/www/js/control/ProfileSettings.jsx b/www/js/control/ProfileSettings.jsx index 566c100c1..f86723e46 100644 --- a/www/js/control/ProfileSettings.jsx +++ b/www/js/control/ProfileSettings.jsx @@ -23,6 +23,7 @@ import { resetDataAndRefresh } from "../config/dynamicConfig"; import { AppContext } from "../App"; import { shareQR } from "../components/QrCode"; import { storageClear } from "../plugin/storage"; +import { getAppVersion } from "../plugin/clientStats"; //any pure functions can go outside const ProfileSettings = () => { @@ -96,7 +97,7 @@ const ProfileSettings = () => { getOPCode(); getSyncSettings(); getConnectURL(); - setAppVersion(ClientStats.getAppVersion()); + setAppVersion(getAppVersion()); } //previously not loaded on regular refresh, this ensures it stays caught up diff --git a/www/js/controllers.js b/www/js/controllers.js index 7efc26c09..56e5c984a 100644 --- a/www/js/controllers.js +++ b/www/js/controllers.js @@ -1,6 +1,7 @@ 'use strict'; import angular from 'angular'; +import { addStatError, addStatReading, statKeys } from './plugin/clientStats'; angular.module('emission.controllers', ['emission.splash.startprefs', 'emission.splash.pushnotify', @@ -23,22 +24,19 @@ angular.module('emission.controllers', ['emission.splash.startprefs', $rootScope.$on('$stateChangeSuccess', function(event, toState, toParams, fromState, fromParams){ console.log("Finished changing state from "+JSON.stringify(fromState) + " to "+JSON.stringify(toState)); - ClientStats.addReading(ClientStats.getStatKeys().STATE_CHANGED, - fromState.name + '-2-' + toState.name).then(function() {}, function() {}); + addStatReading(statKeys.STATE_CHANGED, fromState.name + '-2-' + toState.name); }); $rootScope.$on('$stateChangeError', function(event, toState, toParams, fromState, fromParams, error){ console.log("Error "+error+" while changing state from "+JSON.stringify(fromState) +" to "+JSON.stringify(toState)); - ClientStats.addError(ClientStats.getStatKeys().STATE_CHANGED, - fromState.name + '-2-' + toState.name+ "_" + error).then(function() {}, function() {}); + addStatError(statKeys.STATE_CHANGED, fromState.name + '-2-' + toState.name+ "_" + error); }); $rootScope.$on('$stateNotFound', function(event, unfoundState, fromState, fromParams){ console.log("unfoundState.to = "+unfoundState.to); // "lazy.state" console.log("unfoundState.toParams = " + unfoundState.toParams); // {a:1, b:2} console.log("unfoundState.options = " + unfoundState.options); // {inherit:false} + default options - ClientStats.addError(ClientStats.getStatKeys().STATE_CHANGED, - fromState.name + '-2-' + unfoundState.name).then(function() {}, function() {}); + addStatError(statKeys.STATE_CHANGED, fromState.name + '-2-' + unfoundState.name); }); var isInList = function(element, list) { diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts index 04ca3d539..ba5ffcb4d 100644 --- a/www/js/plugin/storage.ts +++ b/www/js/plugin/storage.ts @@ -1,4 +1,5 @@ import { getAngularService } from "../angular-react-helper"; +import { addStatReading, statKeys } from "./clientStats"; import { displayErrorMsg, logDebug } from "./logger"; const mungeValue = (key, value) => { @@ -158,7 +159,7 @@ export function storageSyncLocalAndNative() { logDebug("STORAGE_PLUGIN: Syncing all missing keys " + allMissing); allMissing.forEach(getUnifiedValue); if (allMissing.length != 0) { - ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + addStatReading(statKeys.MISSING_KEYS, { "type": "local_storage_mismatch", "allMissingLength": allMissing.length, "missingWebLength": missingWeb.length, @@ -172,7 +173,7 @@ export function storageSyncLocalAndNative() { const listAllKeys = window['cordova'].plugins.BEMUserCache.listAllUniqueKeys().then((nativeKeys) => { logDebug("STORAGE_PLUGIN: For the record, all unique native keys are " + nativeKeys); if (nativeKeys.length == 0) { - ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + addStatReading(statKeys.MISSING_KEYS, { "type": "all_native", }).then(logDebug("Logged all missing native keys to client stats")); } diff --git a/www/js/splash/notifScheduler.js b/www/js/splash/notifScheduler.js index 821b6fb09..e062af8d2 100644 --- a/www/js/splash/notifScheduler.js +++ b/www/js/splash/notifScheduler.js @@ -2,6 +2,7 @@ import angular from 'angular'; import { getConfig } from '../config/dynamicConfig'; +import { addStatReading, statKeys } from '../plugin/clientStats'; angular.module('emission.splash.notifscheduler', ['emission.services', @@ -247,7 +248,7 @@ angular.module('emission.splash.notifscheduler', const { reminder_assignment, reminder_join_date, reminder_time_of_day} = prefs; - ClientStats.addReading(ClientStats.getStatKeys().REMINDER_PREFS, { + addStatReading(statKeys.REMINDER_PREFS, { reminder_assignment, reminder_join_date, reminder_time_of_day diff --git a/www/js/splash/remotenotify.js b/www/js/splash/remotenotify.js index 2074da5b8..289d77532 100644 --- a/www/js/splash/remotenotify.js +++ b/www/js/splash/remotenotify.js @@ -13,6 +13,7 @@ 'use strict'; import angular from 'angular'; +import { addStatEvent, statKeys } from '../plugin/clientStats'; angular.module('emission.splash.remotenotify', ['emission.plugin.logger', 'emission.splash.startprefs', @@ -42,10 +43,9 @@ angular.module('emission.splash.remotenotify', ['emission.plugin.logger', remoteNotify.init = function() { $rootScope.$on('cloud:push:notification', function(event, data) { - ClientStats.addEvent(ClientStats.getStatKeys().NOTIFICATION_OPEN).then( - function() { - console.log("Added "+ClientStats.getStatKeys().NOTIFICATION_OPEN+" event. Data = " + JSON.stringify(data)); - }); + addStatEvent(statKeys.NOTIFICATION_OPEN).then(() => { + console.log("Added "+statKeys.NOTIFICATION_OPEN+" event. Data = " + JSON.stringify(data)); + }); Logger.log("data = "+JSON.stringify(data)); if (angular.isDefined(data.additionalData) && angular.isDefined(data.additionalData.payload) && From 25c7077f8783b42904907e38a904017c72836372 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 10:34:56 -0400 Subject: [PATCH 08/25] remove the old ClientStats / clientstats.js Rewritten into a new file clientStats.ts, this old file is not needed anymore. We can remove the file, remove it from index.js, and remove it as a dependency from all of the Angular modules that previously used it. --- www/index.js | 1 - www/js/control/ControlSyncHelper.tsx | 1 - www/js/control/ProfileSettings.jsx | 1 - www/js/controllers.js | 5 +- www/js/plugin/storage.ts | 1 - www/js/splash/notifScheduler.js | 5 +- www/js/splash/remotenotify.js | 5 +- www/js/stats/clientstats.js | 92 ------------------- .../survey/enketo/enketo-add-note-button.js | 3 +- www/js/survey/enketo/enketo-trip-button.js | 3 +- www/js/survey/multilabel/multi-label-ui.js | 3 +- 11 files changed, 9 insertions(+), 111 deletions(-) delete mode 100644 www/js/stats/clientstats.js diff --git a/www/index.js b/www/index.js index a858ef784..89c3a5e26 100644 --- a/www/index.js +++ b/www/index.js @@ -4,7 +4,6 @@ import './css/main.diary.css'; import 'leaflet/dist/leaflet.css'; import './js/ngApp.js'; -import './js/stats/clientstats.js'; import './js/splash/referral.js'; import './js/splash/customURL.js'; import './js/splash/startprefs.js'; diff --git a/www/js/control/ControlSyncHelper.tsx b/www/js/control/ControlSyncHelper.tsx index 992d48562..5acdf5b2d 100644 --- a/www/js/control/ControlSyncHelper.tsx +++ b/www/js/control/ControlSyncHelper.tsx @@ -54,7 +54,6 @@ type syncConfig = { sync_interval: number, export const ForceSyncRow = ({getState}) => { const { t } = useTranslation(); const { colors } = useTheme(); - const ClientStats = getAngularService('ClientStats'); const Logger = getAngularService('Logger'); const [dataPendingVis, setDataPendingVis] = useState(false); diff --git a/www/js/control/ProfileSettings.jsx b/www/js/control/ProfileSettings.jsx index f86723e46..90c5818ee 100644 --- a/www/js/control/ProfileSettings.jsx +++ b/www/js/control/ProfileSettings.jsx @@ -39,7 +39,6 @@ const ProfileSettings = () => { const EmailHelper = getAngularService('EmailHelper'); const NotificationScheduler = getAngularService('NotificationScheduler'); const ControlHelper = getAngularService('ControlHelper'); - const ClientStats = getAngularService('ClientStats'); const StartPrefs = getAngularService('StartPrefs'); //functions that come directly from an Angular service diff --git a/www/js/controllers.js b/www/js/controllers.js index 56e5c984a..75124efce 100644 --- a/www/js/controllers.js +++ b/www/js/controllers.js @@ -7,8 +7,7 @@ angular.module('emission.controllers', ['emission.splash.startprefs', 'emission.splash.pushnotify', 'emission.splash.storedevicesettings', 'emission.splash.localnotify', - 'emission.splash.remotenotify', - 'emission.stats.clientstats']) + 'emission.splash.remotenotify']) .controller('RootCtrl', function($scope) {}) @@ -16,7 +15,7 @@ angular.module('emission.controllers', ['emission.splash.startprefs', .controller('SplashCtrl', function($scope, $state, $interval, $rootScope, StartPrefs, PushNotify, StoreDeviceSettings, - LocalNotify, RemoteNotify, ClientStats) { + LocalNotify, RemoteNotify) { console.log('SplashCtrl invoked'); // alert("attach debugger!"); // PushNotify.startupInit(); diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts index ba5ffcb4d..87be6de9b 100644 --- a/www/js/plugin/storage.ts +++ b/www/js/plugin/storage.ts @@ -141,7 +141,6 @@ function findMissing(fromKeys, toKeys) { } export function storageSyncLocalAndNative() { - const ClientStats = getAngularService('ClientStats'); console.log("STORAGE_PLUGIN: Called syncAllWebAndNativeValues "); const syncKeys = window['cordova'].plugins.BEMUserCache.listAllLocalStorageKeys().then((nativeKeys) => { console.log("STORAGE_PLUGIN: native plugin returned"); diff --git a/www/js/splash/notifScheduler.js b/www/js/splash/notifScheduler.js index e062af8d2..760e797c7 100644 --- a/www/js/splash/notifScheduler.js +++ b/www/js/splash/notifScheduler.js @@ -6,11 +6,10 @@ import { addStatReading, statKeys } from '../plugin/clientStats'; angular.module('emission.splash.notifscheduler', ['emission.services', - 'emission.plugin.logger', - 'emission.stats.clientstats']) + 'emission.plugin.logger']) .factory('NotificationScheduler', function($http, $window, $ionicPlatform, - ClientStats, CommHelper, Logger) { + CommHelper, Logger) { const scheduler = {}; let _config; diff --git a/www/js/splash/remotenotify.js b/www/js/splash/remotenotify.js index 289d77532..c11783a2b 100644 --- a/www/js/splash/remotenotify.js +++ b/www/js/splash/remotenotify.js @@ -16,10 +16,9 @@ import angular from 'angular'; import { addStatEvent, statKeys } from '../plugin/clientStats'; angular.module('emission.splash.remotenotify', ['emission.plugin.logger', - 'emission.splash.startprefs', - 'emission.stats.clientstats']) + 'emission.splash.startprefs']) -.factory('RemoteNotify', function($http, $window, $ionicPopup, $rootScope, ClientStats, +.factory('RemoteNotify', function($http, $window, $ionicPopup, $rootScope, CommHelper, Logger) { var remoteNotify = {}; diff --git a/www/js/stats/clientstats.js b/www/js/stats/clientstats.js deleted file mode 100644 index 7fe4d9cb3..000000000 --- a/www/js/stats/clientstats.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict'; - -import angular from 'angular'; - -angular.module('emission.stats.clientstats', []) - -.factory('ClientStats', function($window) { - var clientStat = {}; - - clientStat.CLIENT_TIME = "stats/client_time"; - clientStat.CLIENT_ERROR = "stats/client_error"; - clientStat.CLIENT_NAV_EVENT = "stats/client_nav_event"; - - clientStat.getStatKeys = function() { - return { - STATE_CHANGED: "state_changed", - BUTTON_FORCE_SYNC: "button_sync_forced", - CHECKED_DIARY: "checked_diary", - DIARY_TIME: "diary_time", - METRICS_TIME: "metrics_time", - CHECKED_INF_SCROLL: "checked_inf_scroll", - INF_SCROLL_TIME: "inf_scroll_time", - VERIFY_TRIP: "verify_trip", - LABEL_TAB_SWITCH: "label_tab_switch", - SELECT_LABEL: "select_label", - EXPANDED_TRIP: "expanded_trip", - NOTIFICATION_OPEN: "notification_open", - REMINDER_PREFS: "reminder_time_prefs", - MISSING_KEYS: "missing_keys" - }; - } - - clientStat.getDB = function() { - if (angular.isDefined($window) && angular.isDefined($window.cordova) && - angular.isDefined($window.cordova.plugins)) { - return $window.cordova.plugins.BEMUserCache; - } else { - return; // undefined - } - } - - clientStat.getAppVersion = function() { - if (angular.isDefined(clientStat.appVersion)) { - return clientStat.appVersion; - } else { - if (angular.isDefined($window) && angular.isDefined($window.cordova) && - angular.isDefined($window.cordova.getAppVersion)) { - $window.cordova.getAppVersion.getVersionNumber().then(function(version) { - clientStat.appVersion = version; - }); - } - return; - } - } - - clientStat.getStatsEvent = function(name, reading) { - var ts_sec = Date.now() / 1000; - var appVersion = clientStat.getAppVersion(); - return { - 'name': name, - 'ts': ts_sec, - 'reading': reading, - 'client_app_version': appVersion, - 'client_os_version': $window.device.version - }; - } - clientStat.addReading = function(name, reading) { - var db = clientStat.getDB(); - if (angular.isDefined(db)) { - return db.putMessage(clientStat.CLIENT_TIME, - clientStat.getStatsEvent(name, reading)); - } - } - - clientStat.addEvent = function(name) { - var db = clientStat.getDB(); - if (angular.isDefined(db)) { - return db.putMessage(clientStat.CLIENT_NAV_EVENT, - clientStat.getStatsEvent(name, null)); - } - } - - clientStat.addError = function(name, errorStr) { - var db = clientStat.getDB(); - if (angular.isDefined(db)) { - return db.putMessage(clientStat.CLIENT_ERROR, - clientStat.getStatsEvent(name, errorStr)); - } - } - - return clientStat; -}) diff --git a/www/js/survey/enketo/enketo-add-note-button.js b/www/js/survey/enketo/enketo-add-note-button.js index a2f0d1557..49f7747f6 100644 --- a/www/js/survey/enketo/enketo-add-note-button.js +++ b/www/js/survey/enketo/enketo-add-note-button.js @@ -5,8 +5,7 @@ import angular from 'angular'; angular.module('emission.survey.enketo.add-note-button', - ['emission.stats.clientstats', - 'emission.services', + ['emission.services', 'emission.survey.enketo.answer', 'emission.survey.inputmatcher']) .factory("EnketoNotesButtonService", function(InputMatcher, EnketoSurveyAnswer, Logger, $timeout) { diff --git a/www/js/survey/enketo/enketo-trip-button.js b/www/js/survey/enketo/enketo-trip-button.js index 5b385a1ac..6e710435f 100644 --- a/www/js/survey/enketo/enketo-trip-button.js +++ b/www/js/survey/enketo/enketo-trip-button.js @@ -14,8 +14,7 @@ import angular from 'angular'; angular.module('emission.survey.enketo.trip.button', - ['emission.stats.clientstats', - 'emission.survey.enketo.answer', + ['emission.survey.enketo.answer', 'emission.survey.inputmatcher']) .factory("EnketoTripButtonService", function(InputMatcher, EnketoSurveyAnswer, Logger, $timeout) { var etbs = {}; diff --git a/www/js/survey/multilabel/multi-label-ui.js b/www/js/survey/multilabel/multi-label-ui.js index 313c8a3a9..7d1bc4007 100644 --- a/www/js/survey/multilabel/multi-label-ui.js +++ b/www/js/survey/multilabel/multi-label-ui.js @@ -3,8 +3,7 @@ import { baseLabelInputDetails, getBaseLabelInputs, getFakeEntry, getLabelInputD import { getConfig } from '../../config/dynamicConfig'; angular.module('emission.survey.multilabel.buttons', - ['emission.stats.clientstats', - 'emission.survey.inputmatcher']) + ['emission.survey.inputmatcher']) .factory("MultiLabelService", function($rootScope, InputMatcher, $timeout, $ionicPlatform, Logger) { var mls = {}; From 1a476343c4ae8ca5dc90c10d46cf7033e5d2e846 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 13:05:40 -0400 Subject: [PATCH 09/25] fix getAppVersion in clientstats This is used in ProfileSettings is for the "App Version" row at the very bottom of the Profile page. getAppVersion() was not working correctly - the correct function to call is `getVersionNumber` which returns a promise. In a .then block, we can memoize this value in the local let appVersion. Since it is memoized, we don't need a dedicated state value in ProfileSettings for appVersion. We can just call getAppVersion() directly where it's used in the SettingRow. --- www/js/control/ProfileSettings.jsx | 4 +--- www/js/plugin/clientStats.ts | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/www/js/control/ProfileSettings.jsx b/www/js/control/ProfileSettings.jsx index 90c5818ee..03523f376 100644 --- a/www/js/control/ProfileSettings.jsx +++ b/www/js/control/ProfileSettings.jsx @@ -69,7 +69,6 @@ const ProfileSettings = () => { const [syncSettings, setSyncSettings] = useState({}); const [cacheResult, setCacheResult] = useState(""); const [connectSettings, setConnectSettings] = useState({}); - const [appVersion, setAppVersion] = useState(""); const [uiConfig, setUiConfig] = useState({}); const [consentDoc, setConsentDoc] = useState({}); const [dumpDate, setDumpDate] = useState(new Date()); @@ -96,7 +95,6 @@ const ProfileSettings = () => { getOPCode(); getSyncSettings(); getConnectURL(); - setAppVersion(getAppVersion()); } //previously not loaded on regular refresh, this ensures it stays caught up @@ -376,7 +374,7 @@ const ProfileSettings = () => { - console.log("")} desc={appVersion}> + console.log("")} desc={getAppVersion()}> {/* menu for "nuke data" */} diff --git a/www/js/plugin/clientStats.ts b/www/js/plugin/clientStats.ts index b945ee5f8..746672dc7 100644 --- a/www/js/plugin/clientStats.ts +++ b/www/js/plugin/clientStats.ts @@ -21,8 +21,10 @@ export const statKeys = { let appVersion; export const getAppVersion = () => { - appVersion ||= window['cordova']?.plugins?.BEMUserCache?.getAppVersion(); - return appVersion; + if (appVersion) return appVersion; + window['cordova']?.getAppVersion.getVersionNumber().then((version) => { + appVersion = version; + }); } export const getStatsEvent = (name: string, reading: any) => { From 598d1fb8ef2b8c4ce773b715474b5c636c2004df Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 13:31:34 -0400 Subject: [PATCH 10/25] add some cordova mocks: cordova,device,appversion There are a few other 'window' variables that are provided in a Cordova/Ionic app and are expected in various few places throughout the codebase. To test, we need to be able to mock these too. mockCordova and mockDevice just provide fake platform/version info. mockGetAppVersion mocks the cordova-plugin-app-version, which is a third-party plugin we use to get the version of the app. This is used in clientStats.ts, so we will need to use this mock when we test that. The mock returns info for a "Mock App", version 1.2.3. --- www/__mocks__/cordovaMocks.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 5f723a5d7..f7d0a6ec6 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -1,3 +1,27 @@ +export const mockCordova = () => { + window['cordova'] ||= {}; + window['cordova'].platformId ||= 'ios'; + window['cordova'].platformVersion ||= '6.2.0'; + window['cordova'].plugins ||= {}; +} + +export const mockDevice = () => { + window['device'] ||= {}; + window['device'].platform ||= 'ios'; + window['device'].version ||= '14.0.0'; +} + +export const mockGetAppVersion = () => { + const mockGetAppVersion = { + getAppName: () => new Promise((rs, rj) => setTimeout(() => rs('Mock App'), 10)), + getPackageName: () => new Promise((rs, rj) => setTimeout(() => rs('com.example.mockapp'), 10)), + getVersionCode: () => new Promise((rs, rj) => setTimeout(() => rs('123'), 10)), + getVersionNumber: () => new Promise((rs, rj) => setTimeout(() => rs('1.2.3'), 10)), + } + window['cordova'] ||= {}; + window['cordova'].getAppVersion = mockGetAppVersion; +} + export const mockBEMUserCache = () => { const _cache = {}; const mockBEMUserCache = { From 47aceaf986cf281c74b21c73c685f5a7ab3b66a8 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 14:50:22 -0400 Subject: [PATCH 11/25] make getAppVersion work async getAppVersion() calls cordova-plugin-app-version's getVersionNumber, which is asynchronous. So getAppVersion should be asynchronous too. I changed it to return a promise. We handle this in clientStats by adopting async/await syntax. We handle this in ProfileSettings by storing it as a ref once the promise is resolved. Then we just access it as appVersion.current. --- www/js/control/ProfileSettings.jsx | 8 ++++++-- www/js/plugin/clientStats.ts | 24 ++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/www/js/control/ProfileSettings.jsx b/www/js/control/ProfileSettings.jsx index 03523f376..4a263acc5 100644 --- a/www/js/control/ProfileSettings.jsx +++ b/www/js/control/ProfileSettings.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useContext } from "react"; +import React, { useState, useEffect, useContext, useRef } from "react"; import { Modal, StyleSheet, ScrollView } from "react-native"; import { Dialog, Button, useTheme, Text, Appbar, IconButton } from "react-native-paper"; import { getAngularService } from "../angular-react-helper"; @@ -72,6 +72,7 @@ const ProfileSettings = () => { const [uiConfig, setUiConfig] = useState({}); const [consentDoc, setConsentDoc] = useState({}); const [dumpDate, setDumpDate] = useState(new Date()); + const appVersion = useRef(); let carbonDatasetString = t('general-settings.carbon-dataset') + ": " + CarbonDatasetHelper.getCurrentCarbonDatasetCode(); const carbonOptions = CarbonDatasetHelper.getCarbonDatasetOptions(); @@ -95,6 +96,9 @@ const ProfileSettings = () => { getOPCode(); getSyncSettings(); getConnectURL(); + getAppVersion().then((version) => { + appVersion.current = version; + }); } //previously not loaded on regular refresh, this ensures it stays caught up @@ -374,7 +378,7 @@ const ProfileSettings = () => { - console.log("")} desc={getAppVersion()}> + console.log("")} desc={appVersion.current}> {/* menu for "nuke data" */} diff --git a/www/js/plugin/clientStats.ts b/www/js/plugin/clientStats.ts index 746672dc7..1e06208eb 100644 --- a/www/js/plugin/clientStats.ts +++ b/www/js/plugin/clientStats.ts @@ -21,30 +21,34 @@ export const statKeys = { let appVersion; export const getAppVersion = () => { - if (appVersion) return appVersion; - window['cordova']?.getAppVersion.getVersionNumber().then((version) => { + if (appVersion) return Promise.resolve(appVersion); + return window['cordova']?.getAppVersion.getVersionNumber().then((version) => { appVersion = version; + return version; }); } -export const getStatsEvent = (name: string, reading: any) => { +const getStatsEvent = async (name: string, reading: any) => { const ts = Date.now() / 1000; - const client_app_version = getAppVersion(); + const client_app_version = await getAppVersion(); const client_os_version = window['device'].version; return { name, ts, reading, client_app_version, client_os_version }; } -export const addStatReading = (name: string, reading: any) => { +export const addStatReading = async (name: string, reading: any) => { const db = window['cordova']?.plugins?.BEMUserCache; - if (db) return db.putMessage(CLIENT_TIME, getStatsEvent(name, reading)); + const event = await getStatsEvent(name, reading); + if (db) return db.putMessage(CLIENT_TIME, event); } -export const addStatEvent = (name: string) => { +export const addStatEvent = async (name: string) => { const db = window['cordova']?.plugins?.BEMUserCache; - if (db) return db.putMessage(CLIENT_NAV_EVENT, getStatsEvent(name, null)); + const event = await getStatsEvent(name, null); + if (db) return db.putMessage(CLIENT_NAV_EVENT, event); } -export const addStatError = (name: string, errorStr: string) => { +export const addStatError = async (name: string, errorStr: string) => { const db = window['cordova']?.plugins?.BEMUserCache; - if (db) return db.putMessage(CLIENT_ERROR, getStatsEvent(name, errorStr)); + const event = await getStatsEvent(name, errorStr); + if (db) return db.putMessage(CLIENT_ERROR, event); } From c7b06b7e4796bdf6ee6c5cd00bd7f2e2e28697d0 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 14:58:30 -0400 Subject: [PATCH 12/25] add tests for clientStats.ts Tests the methods exported from clientStats.ts, including getAppVersion, addStatReading, addStatEvent, and addStatError. The tests record these stats and then query the usercache for the messages, expecting to see a new entry filled in with the fake data. putMessage and getAllMessages needed to be added to the mockBEMUserCache, since the client stats are recorded as 'messages' rather than 'key-value' pairs --- www/__mocks__/cordovaMocks.ts | 16 ++++++++++ www/__tests__/clientStats.test.ts | 52 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 www/__tests__/clientStats.test.ts diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index f7d0a6ec6..44c21677c 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -24,6 +24,7 @@ export const mockGetAppVersion = () => { export const mockBEMUserCache = () => { const _cache = {}; + const messages = []; const mockBEMUserCache = { getLocalStorage: (key: string, isSecure: boolean) => { return new Promise((rs, rj) => @@ -69,6 +70,21 @@ export const mockBEMUserCache = () => { rs(Object.keys(_cache)); }, 100) ); + }, + putMessage: (key: string, value: any) => { + return new Promise((rs, rj) => + setTimeout(() => { + messages.push({ key, value }); + rs(); + }, 100) + ); + }, + getAllMessages: (key: string, withMetadata?: boolean) => { + return new Promise((rs, rj) => + setTimeout(() => { + rs(messages.filter(m => m.key == key).map(m => m.value)); + }, 100) + ); } } window['cordova'] ||= {}; diff --git a/www/__tests__/clientStats.test.ts b/www/__tests__/clientStats.test.ts new file mode 100644 index 000000000..d1a054195 --- /dev/null +++ b/www/__tests__/clientStats.test.ts @@ -0,0 +1,52 @@ +import { mockBEMUserCache, mockDevice, mockGetAppVersion } from "../__mocks__/cordovaMocks"; +import { addStatError, addStatEvent, addStatReading, getAppVersion, statKeys } from "../js/plugin/clientStats"; + +mockDevice(); +// this mocks cordova-plugin-app-version, generating a "Mock App", version "1.2.3" +mockGetAppVersion(); +// clientStats.ts uses BEMUserCache to store the stats, so we need to mock that too +mockBEMUserCache(); +const db = window['cordova']?.plugins?.BEMUserCache; + +it('gets the app version', async () => { + const ver = await getAppVersion(); + expect(ver).toEqual('1.2.3'); +}); + +it('stores a client stats reading', async () => { + const reading = { a: 1, b: 2 }; + await addStatReading(statKeys.REMINDER_PREFS, reading); + const storedMessages = await db.getAllMessages('stats/client_time', false); + expect(storedMessages).toContainEqual({ + name: statKeys.REMINDER_PREFS, + ts: expect.any(Number), + reading, + client_app_version: '1.2.3', + client_os_version: '14.0.0' + }); +}); + +it('stores a client stats event', async () => { + await addStatEvent(statKeys.BUTTON_FORCE_SYNC); + const storedMessages = await db.getAllMessages('stats/client_nav_event', false); + expect(storedMessages).toContainEqual({ + name: statKeys.BUTTON_FORCE_SYNC, + ts: expect.any(Number), + reading: null, + client_app_version: '1.2.3', + client_os_version: '14.0.0' + }); +}); + +it('stores a client stats error', async () => { + const errorStr = 'test error'; + await addStatError(statKeys.MISSING_KEYS, errorStr); + const storedMessages = await db.getAllMessages('stats/client_error', false); + expect(storedMessages).toContainEqual({ + name: statKeys.MISSING_KEYS, + ts: expect.any(Number), + reading: errorStr, + client_app_version: '1.2.3', + client_os_version: '14.0.0' + }); +}); From c16bde5ed53e68616efc5d0e50b3ff7620685a30 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 16:21:12 -0400 Subject: [PATCH 13/25] rewrite CommHelper service into commHelper.ts CommHelper from js/services.js is being rewritten into commHelper.ts. This commit moves over the functions from CommHelper that we need. So now the following functions are exported: - fetchUrlCached (was already in commHelper.ts) - getRawEntries - getPipelineRangeTs - getPipelineCompleteTs - getMetrics - getAggregateData - registerUser - updateUser - getUser The following functions were not moved over to the new commHelper.ts because they are not currently used in the codebase and I don't anticipate them being used in the foreseeable future: - putOne - getTimelineForDay - habiticaRegister - habiticaProxy - moment2Localdate - moment2Timestamp - getRawEntriesForLocalDate --- www/js/commHelper.ts | 124 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/www/js/commHelper.ts b/www/js/commHelper.ts index 074093999..d0042b6ab 100644 --- a/www/js/commHelper.ts +++ b/www/js/commHelper.ts @@ -17,3 +17,127 @@ export async function fetchUrlCached(url) { logDebug(`fetchUrlCached: fetched data for url ${url}, returning`); return text; } + +function processErrorMessages(errorMsg) { + if (errorMsg.includes("403")) { + errorMsg = "Error: OPcode does not exist on the server. " + errorMsg; + console.error("Error 403 found. " + errorMsg); + } + return errorMsg; +} + +export function getRawEntries(key_list, start_ts, end_ts, time_key = "metadata.write_ts", + max_entries = undefined, trunc_method = "sample") { + return new Promise((rs, rj) => { + const msgFiller = (message) => { + message.key_list = key_list; + message.start_time = start_ts; + message.end_time = end_ts; + message.key_time = time_key; + if (max_entries !== undefined) { + message.max_entries = max_entries; + message.trunc_method = trunc_method; + } + logDebug(`About to return message ${JSON.stringify(message)}`); + } + logDebug("getRawEntries: about to get pushGetJSON for the timestamp"); + window['cordova'].plugins.BEMServerComm.pushGetJSON("/datastreams/find_entries/timestamp", msgFiller, rs, rj); + }).catch(error => { + error = `While getting raw entries, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function getPipelineRangeTs() { + return new Promise((rs, rj) => { + logDebug("getting pipeline range timestamps"); + window['cordova'].plugins.BEMServerComm.getUserPersonalData("/pipeline/get_range_ts", rs, rj); + }).catch(error => { + error = `While getting pipeline range timestamps, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function getPipelineCompleteTs() { + return new Promise((rs, rj) => { + logDebug("getting pipeline complete timestamp"); + window['cordova'].plugins.BEMServerComm.getUserPersonalData("/pipeline/get_complete_ts", rs, rj); + }).catch(error => { + error = `While getting pipeline complete timestamp, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function getMetrics(timeType: 'timestamp'|'local_date', metricsQuery) { + return new Promise((rs, rj) => { + const msgFiller = (message) => { + for (let key in metricsQuery) { + message[key] = metricsQuery[key]; + } + } + window['cordova'].plugins.BEMServerComm.pushGetJSON(`/result/metrics/${timeType}`, msgFiller, rs, rj); + }).catch(error => { + error = `While getting metrics, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function getAggregateData(path: string, data: any) { + return new Promise((rs, rj) => { + const fullUrl = `${window['$rootScope'].connectUrl}/${path}`; + data["aggregate"] = true; + + if (window['$rootScope'].aggregateAuth === "no_auth") { + logDebug(`getting aggregate data without user authentication from ${fullUrl} with arguments ${JSON.stringify(data)}`); + const options = { + method: 'post', + data: data, + responseType: 'json' + } + window['cordova'].plugin.http.sendRequest(fullUrl, options, + (response) => { + rs(response.data); + }, (error) => { + rj(error); + }); + } else { + logDebug(`getting aggregate data with user authentication from ${fullUrl} with arguments ${JSON.stringify(data)}`); + const msgFiller = (message) => { + return Object.assign(message, data); + } + window['cordova'].plugins.BEMServerComm.pushGetJSON(`/${path}`, msgFiller, rs, rj); + } + }).catch(error => { + error = `While getting aggregate data, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function registerUser(successCallback, errorCallback) { + window['cordova'].plugins.BEMServerComm.getUserPersonalData("/profile/create", successCallback, errorCallback); +} + +export function updateUser(updateDoc) { + return new Promise((rs, rj) => { + window['cordova'].plugins.BEMServerComm.postUserPersonalData("/profile/update", "update_doc", updateDoc, rs, rj); + }).catch(error => { + error = `While updating user, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} + +export function getUser() { + return new Promise((rs, rj) => { + window['cordova'].plugins.BEMServerComm.getUserPersonalData("/profile/get", rs, rj); + }).catch(error => { + error = `While getting user, ${error}`; + error = processErrorMessages(error); + throw(error); + }); +} From 1dc7451cad497bd093c58f26f2a5b388984fd20f Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 16:24:04 -0400 Subject: [PATCH 14/25] use commHelper.ts everywhere, not old CommHelper CommHelper from js/services.js was rewritten into js/commHelper.ts. This commit switches all functions calls to the new commHelper.ts. All of the functions are named the same and perform the same logic. --- www/js/control/ControlSyncHelper.tsx | 3 ++- www/js/diary/LabelTab.tsx | 3 ++- www/js/diary/services.js | 3 ++- www/js/metrics/MetricsTab.tsx | 5 +++-- www/js/onboarding/SaveQrPage.tsx | 3 ++- www/js/services.js | 7 ++++--- www/js/splash/notifScheduler.js | 5 +++-- www/js/splash/pushnotify.js | 3 ++- www/js/splash/storedevicesettings.js | 3 ++- 9 files changed, 22 insertions(+), 13 deletions(-) diff --git a/www/js/control/ControlSyncHelper.tsx b/www/js/control/ControlSyncHelper.tsx index 5acdf5b2d..84e1effba 100644 --- a/www/js/control/ControlSyncHelper.tsx +++ b/www/js/control/ControlSyncHelper.tsx @@ -9,6 +9,7 @@ import SettingRow from "./SettingRow"; import AlertBar from "./AlertBar"; import moment from "moment"; import { addStatEvent, statKeys } from "../plugin/clientStats"; +import { updateUser } from "../commHelper"; /* * BEGIN: Simple read/write wrappers @@ -214,7 +215,7 @@ const ControlSyncHelper = ({ editVis, setEditVis }) => { try{ let set = setConfig(localConfig); //NOTE -- we need to make sure we update these settings in ProfileSettings :) -- getting rid of broadcast handling for migration!! - CommHelper.updateUser({ + updateUser({ // TODO: worth thinking about where best to set this // Currently happens in native code. Now that we are switching // away from parse, we can store this from javascript here. diff --git a/www/js/diary/LabelTab.tsx b/www/js/diary/LabelTab.tsx index 42b173017..f8c63da72 100644 --- a/www/js/diary/LabelTab.tsx +++ b/www/js/diary/LabelTab.tsx @@ -22,6 +22,7 @@ import { SurveyOptions } from "../survey/survey"; import { getLabelOptions } from "../survey/multilabel/confirmHelper"; import { displayError } from "../plugin/logger"; import { useTheme } from "react-native-paper"; +import { getPipelineRangeTs } from "../commHelper"; let labelPopulateFactory, labelsResultMap, notesResultMap, showPlaces; const ONE_DAY = 24 * 60 * 60; // seconds @@ -105,7 +106,7 @@ const LabelTab = () => { async function loadTimelineEntries() { try { - const pipelineRange = await CommHelper.getPipelineRangeTs(); + const pipelineRange = await getPipelineRangeTs(); [labelsResultMap, notesResultMap] = await getAllUnprocessedInputs(pipelineRange, labelPopulateFactory, enbs); Logger.log("After reading unprocessedInputs, labelsResultMap =" + JSON.stringify(labelsResultMap) + "; notesResultMap = " + JSON.stringify(notesResultMap)); diff --git a/www/js/diary/services.js b/www/js/diary/services.js index c9dfd1bbf..1e877107b 100644 --- a/www/js/diary/services.js +++ b/www/js/diary/services.js @@ -4,6 +4,7 @@ import angular from 'angular'; import { getBaseModeByKey, getBaseModeOfLabeledTrip } from './diaryHelper'; import { SurveyOptions } from '../survey/survey'; import { getConfig } from '../config/dynamicConfig'; +import { getRawEntries } from '../commHelper'; angular.module('emission.main.diary.services', ['emission.plugin.logger', 'emission.services']) @@ -41,7 +42,7 @@ angular.module('emission.main.diary.services', ['emission.plugin.logger', template: i18next.t('service.reading-server') }); const readPromises = [ - CommHelper.getRawEntries(["analysis/composite_trip"], + getRawEntries(["analysis/composite_trip"], startTs, endTs, "data.end_ts"), ]; return Promise.all(readPromises) diff --git a/www/js/metrics/MetricsTab.tsx b/www/js/metrics/MetricsTab.tsx index 748db2b99..726d59800 100644 --- a/www/js/metrics/MetricsTab.tsx +++ b/www/js/metrics/MetricsTab.tsx @@ -16,6 +16,7 @@ import Carousel from "../components/Carousel"; import DailyActiveMinutesCard from "./DailyActiveMinutesCard"; import CarbonTextCard from "./CarbonTextCard"; import ActiveMinutesTableCard from "./ActiveMinutesTableCard"; +import { getAggregateData, getMetrics } from "../commHelper"; export const METRIC_LIST = ['duration', 'mean_speed', 'count', 'distance'] as const; @@ -29,8 +30,8 @@ async function fetchMetricsFromServer(type: 'user'|'aggregate', dateRange: DateT is_return_aggregate: (type == 'aggregate'), } if (type == 'user') - return CommHelper.getMetrics('timestamp', query); - return CommHelper.getAggregateData("result/metrics/timestamp", query); + return getMetrics('timestamp', query); + return getAggregateData("result/metrics/timestamp", query); } function getLastTwoWeeksDtRange() { diff --git a/www/js/onboarding/SaveQrPage.tsx b/www/js/onboarding/SaveQrPage.tsx index 57fe8f679..6d4678689 100644 --- a/www/js/onboarding/SaveQrPage.tsx +++ b/www/js/onboarding/SaveQrPage.tsx @@ -11,6 +11,7 @@ import QrCode, { shareQR } from "../components/QrCode"; import { onboardingStyles } from "./OnboardingStack"; import { preloadDemoSurveyResponse } from "./SurveyPage"; import { storageSet } from "../plugin/storage"; +import { registerUser } from "../commHelper"; const SaveQrPage = ({ }) => { @@ -37,7 +38,7 @@ const SaveQrPage = ({ }) => { const EXPECTED_METHOD = "prompted-auth"; const dbStorageObject = {"token": token}; return storageSet(EXPECTED_METHOD, dbStorageObject).then((r) => { - CommHelper.registerUser((successResult) => { + registerUser((successResult) => { refreshOnboardingState(); }, function(errorResult) { displayError(errorResult, "User registration error"); diff --git a/www/js/services.js b/www/js/services.js index 1955cd4bb..ba9b31784 100644 --- a/www/js/services.js +++ b/www/js/services.js @@ -1,6 +1,7 @@ 'use strict'; import angular from 'angular'; +import { getRawEntries } from './commHelper'; angular.module('emission.services', ['emission.plugin.logger']) @@ -345,7 +346,7 @@ angular.module('emission.services', ['emission.plugin.logger']) // Probably in www/json... this.getUnifiedSensorDataForInterval = function(key, tq) { var localPromise = $window.cordova.plugins.BEMUserCache.getSensorDataForInterval(key, tq, true); - var remotePromise = CommHelper.getRawEntries([key], tq.startTs, tq.endTs) + var remotePromise = getRawEntries([key], tq.startTs, tq.endTs) .then(function(serverResponse) { return serverResponse.phone_data; }); @@ -354,7 +355,7 @@ angular.module('emission.services', ['emission.plugin.logger']) this.getUnifiedMessagesForInterval = function(key, tq, withMetadata) { var localPromise = $window.cordova.plugins.BEMUserCache.getMessagesForInterval(key, tq, true); - var remotePromise = CommHelper.getRawEntries([key], tq.startTs, tq.endTs) + var remotePromise = getRawEntries([key], tq.startTs, tq.endTs) .then(function(serverResponse) { return serverResponse.phone_data; }); @@ -456,7 +457,7 @@ angular.module('emission.services', ['emission.plugin.logger']) }); }; - CommHelper.getRawEntries(null, startMoment.unix(), endMoment.unix()) + getRawEntries(null, startMoment.unix(), endMoment.unix()) .then(writeDumpFile) .then(emailData) .then(function() { diff --git a/www/js/splash/notifScheduler.js b/www/js/splash/notifScheduler.js index 760e797c7..530439df8 100644 --- a/www/js/splash/notifScheduler.js +++ b/www/js/splash/notifScheduler.js @@ -3,6 +3,7 @@ import angular from 'angular'; import { getConfig } from '../config/dynamicConfig'; import { addStatReading, statKeys } from '../plugin/clientStats'; +import { getUser, updateUser } from '../commHelper'; angular.module('emission.splash.notifscheduler', ['emission.services', @@ -219,7 +220,7 @@ angular.module('emission.splash.notifscheduler', */ scheduler.getReminderPrefs = async () => { - const user = await CommHelper.getUser(); + const user = await getUser(); if (user?.reminder_assignment && user?.reminder_join_date && user?.reminder_time_of_day) { @@ -232,7 +233,7 @@ angular.module('emission.splash.notifscheduler', } scheduler.setReminderPrefs = async (newPrefs) => { - await CommHelper.updateUser(newPrefs) + await updateUser(newPrefs) const updatePromise = new Promise((resolve, reject) => { //enforcing update before moving on update().then(() => { diff --git a/www/js/splash/pushnotify.js b/www/js/splash/pushnotify.js index 66c70f45c..f846e41ed 100644 --- a/www/js/splash/pushnotify.js +++ b/www/js/splash/pushnotify.js @@ -14,6 +14,7 @@ */ import angular from 'angular'; +import { updateUser } from '../commHelper'; angular.module('emission.splash.pushnotify', ['emission.plugin.logger', 'emission.services', @@ -86,7 +87,7 @@ angular.module('emission.splash.pushnotify', ['emission.plugin.logger', console.log("Got error "+error+" while reading config, returning default = 3600"); return 3600; }).then(function(sync_interval) { - CommHelper.updateUser({ + updateUser({ device_token: t.token, curr_platform: ionic.Platform.platform(), curr_sync_interval: sync_interval diff --git a/www/js/splash/storedevicesettings.js b/www/js/splash/storedevicesettings.js index aaaf82c6b..c9c89d5de 100644 --- a/www/js/splash/storedevicesettings.js +++ b/www/js/splash/storedevicesettings.js @@ -1,4 +1,5 @@ import angular from 'angular'; +import { updateUser } from '../commHelper'; angular.module('emission.splash.storedevicesettings', ['emission.plugin.logger', 'emission.services', @@ -21,7 +22,7 @@ angular.module('emission.splash.storedevicesettings', ['emission.plugin.logger', client_app_version: appver }; Logger.log("About to update profile with settings = "+JSON.stringify(updateJSON)); - return CommHelper.updateUser(updateJSON); + return updateUser(updateJSON); }).then(function(updateJSON) { // alert("Finished saving token = "+JSON.stringify(t.token)); }).catch(function(error) { From f78dab19e781fa1a42688193ba2ce868fc229291 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 28 Sep 2023 16:27:55 -0400 Subject: [PATCH 15/25] remove the old CommHelper service Rewritten into commHelper.ts, this Angular service is not needed anymore. We can remove the 'factory' declaration from js/services.js and remove it as a dependency from all of the Angular modules that previously used it. --- www/js/control/ControlSyncHelper.tsx | 1 - www/js/diary/LabelTab.tsx | 1 - www/js/diary/services.js | 2 +- www/js/metrics/MetricsTab.tsx | 1 - www/js/onboarding/SaveQrPage.tsx | 1 - www/js/services.js | 263 +-------------------------- www/js/splash/notifScheduler.js | 3 +- www/js/splash/pushnotify.js | 2 +- www/js/splash/remotenotify.js | 3 +- www/js/splash/storedevicesettings.js | 2 +- 10 files changed, 6 insertions(+), 273 deletions(-) diff --git a/www/js/control/ControlSyncHelper.tsx b/www/js/control/ControlSyncHelper.tsx index 84e1effba..edc0e7470 100644 --- a/www/js/control/ControlSyncHelper.tsx +++ b/www/js/control/ControlSyncHelper.tsx @@ -182,7 +182,6 @@ export const ForceSyncRow = ({getState}) => { const ControlSyncHelper = ({ editVis, setEditVis }) => { const { t } = useTranslation(); const { colors } = useTheme(); - const CommHelper = getAngularService("CommHelper"); const Logger = getAngularService("Logger"); const [ localConfig, setLocalConfig ] = useState(); diff --git a/www/js/diary/LabelTab.tsx b/www/js/diary/LabelTab.tsx index f8c63da72..bb2430481 100644 --- a/www/js/diary/LabelTab.tsx +++ b/www/js/diary/LabelTab.tsx @@ -49,7 +49,6 @@ const LabelTab = () => { const $ionicPopup = getAngularService('$ionicPopup'); const Logger = getAngularService('Logger'); const Timeline = getAngularService('Timeline'); - const CommHelper = getAngularService('CommHelper'); const enbs = getAngularService('EnketoNotesButtonService'); // initialization, once the appConfig is loaded diff --git a/www/js/diary/services.js b/www/js/diary/services.js index 1e877107b..774273fa2 100644 --- a/www/js/diary/services.js +++ b/www/js/diary/services.js @@ -8,7 +8,7 @@ import { getRawEntries } from '../commHelper'; angular.module('emission.main.diary.services', ['emission.plugin.logger', 'emission.services']) -.factory('Timeline', function(CommHelper, $http, $ionicLoading, $ionicPlatform, $window, +.factory('Timeline', function($http, $ionicLoading, $ionicPlatform, $window, $rootScope, UnifiedDataLoader, Logger, $injector) { var timeline = {}; // corresponds to the old $scope.data. Contains all state for the current diff --git a/www/js/metrics/MetricsTab.tsx b/www/js/metrics/MetricsTab.tsx index 726d59800..450155622 100644 --- a/www/js/metrics/MetricsTab.tsx +++ b/www/js/metrics/MetricsTab.tsx @@ -21,7 +21,6 @@ import { getAggregateData, getMetrics } from "../commHelper"; export const METRIC_LIST = ['duration', 'mean_speed', 'count', 'distance'] as const; async function fetchMetricsFromServer(type: 'user'|'aggregate', dateRange: DateTime[]) { - const CommHelper = getAngularService('CommHelper'); const query = { freq: 'D', start_time: dateRange[0].toSeconds(), diff --git a/www/js/onboarding/SaveQrPage.tsx b/www/js/onboarding/SaveQrPage.tsx index 6d4678689..d8b555f14 100644 --- a/www/js/onboarding/SaveQrPage.tsx +++ b/www/js/onboarding/SaveQrPage.tsx @@ -34,7 +34,6 @@ const SaveQrPage = ({ }) => { }, [overallStatus]); function login(token) { - const CommHelper = getAngularService('CommHelper'); const EXPECTED_METHOD = "prompted-auth"; const dbStorageObject = {"token": token}; return storageSet(EXPECTED_METHOD, dbStorageObject).then((r) => { diff --git a/www/js/services.js b/www/js/services.js index ba9b31784..0c9c6e2ac 100644 --- a/www/js/services.js +++ b/www/js/services.js @@ -5,266 +5,6 @@ import { getRawEntries } from './commHelper'; angular.module('emission.services', ['emission.plugin.logger']) -.service('CommHelper', function($rootScope) { - var getConnectURL = function(successCallback, errorCallback) { - window.cordova.plugins.BEMConnectionSettings.getSettings( - function(settings) { - successCallback(settings.connectUrl); - }, errorCallback); - }; - - var processErrorMessages = function(errorMsg) { - if (errorMsg.includes("403")) { - errorMsg = "Error: OPcode does not exist on the server. " + errorMsg; - console.error("Error 403 found. " + errorMsg); - } - return errorMsg; - } - - this.registerUser = function(successCallback, errorCallback) { - window.cordova.plugins.BEMServerComm.getUserPersonalData("/profile/create", successCallback, errorCallback); - }; - - this.updateUser = function(updateDoc) { - return new Promise(function(resolve, reject) { - window.cordova.plugins.BEMServerComm.postUserPersonalData("/profile/update", "update_doc", updateDoc, resolve, reject); - }) - .catch(error => { - error = "While updating user, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getUser = function() { - return new Promise(function(resolve, reject) { - window.cordova.plugins.BEMServerComm.getUserPersonalData("/profile/get", resolve, reject); - }) - .catch(error => { - error = "While getting user, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.putOne = function(key, data) { - var now = moment().unix(); - var md = { - "write_ts": now, - "read_ts": now, - "time_zone": moment.tz.guess(), - "type": "message", - "key": key, - "platform": ionic.Platform.platform() - }; - var entryToPut = { - "metadata": md, - "data": data - } - return new Promise(function(resolve, reject) { - window.cordova.plugins.BEMServerComm.postUserPersonalData("/usercache/putone", "the_entry", entryToPut, resolve, reject); - }) - .catch(error => { - error = "While putting one entry, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getTimelineForDay = function(date) { - return new Promise(function(resolve, reject) { - var dateString = date.startOf('day').format('YYYY-MM-DD'); - window.cordova.plugins.BEMServerComm.getUserPersonalData("/timeline/getTrips/"+dateString, resolve, reject); - }) - .catch(error => { - error = "While getting timeline for day, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - /* - * var regConfig = {'username': ....} - * Other fields can be added easily and the server can be modified at the same time. - */ - this.habiticaRegister = function(regConfig) { - return new Promise(function(resolve, reject){ - window.cordova.plugins.BEMServerComm.postUserPersonalData("/habiticaRegister", "regConfig", regConfig, resolve, reject); - }); - }; - - /* - * Example usage: - * Get profile: - * callOpts = {'method': 'GET', 'method_url': "/api/v3/user", - 'method_args': null} - * Go to sleep: - * callOpts = {'method': 'POST', 'method_url': "/api/v3/user/sleep", - 'method_args': {'data': True}} - * Stop sleeping: - * callOpts = {'method': 'POST', 'method_url': "/api/v3/user/sleep", - 'method_args': {'data': False}} - * Get challenges for a user: - * callOpts = {'method': 'GET', 'method_url': "/api/v3/challenges/user", - 'method_args': null} - * .... - */ - - this.habiticaProxy = function(callOpts){ - return new Promise(function(resolve, reject){ - window.cordova.plugins.BEMServerComm.postUserPersonalData("/habiticaProxy", "callOpts", callOpts, resolve, reject); - }) - .catch(error => { - error = "While habitica proxy, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getMetrics = function(timeType, metrics_query) { - return new Promise(function(resolve, reject) { - var msgFiller = function(message) { - for (var key in metrics_query) { - message[key] = metrics_query[key] - }; - }; - window.cordova.plugins.BEMServerComm.pushGetJSON("/result/metrics/"+timeType, msgFiller, resolve, reject); - }) - .catch(error => { - error = "While getting metrics, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - /* - * key_list = list of keys to retrieve or None for all keys - * start_time = beginning timestamp for range - * end_time = ending timestamp for rangeA - */ - this.moment2Localdate = function(momentObj) { - return { - year: momentObj.year(), - month: momentObj.month() + 1, - day: momentObj.date(), - }; - }; - - this.moment2Timestamp = function(momentObj) { - return momentObj.unix(); - } - - // time_key is typically metadata.write_ts or data.ts - this.getRawEntriesForLocalDate = function(key_list, start_ts, end_ts, - time_key = "metadata.write_ts", max_entries = undefined, trunc_method = "sample") { - return new Promise(function(resolve, reject) { - var msgFiller = function(message) { - message.key_list = key_list; - message.from_local_date = moment2Localdate(moment.unix(start_ts)); - message.to_local_date = moment2Localdate(moment.unix(end_ts)); - message.key_local_date = time_key; - if (max_entries !== undefined) { - message.max_entries = max_entries; - message.trunc_method = trunc_method; - } - console.log("About to return message "+JSON.stringify(message)); - }; - console.log("getRawEntries: about to get pushGetJSON for the timestamp"); - window.cordova.plugins.BEMServerComm.pushGetJSON("/datastreams/find_entries/local_date", msgFiller, resolve, reject); - }) - .catch(error => { - error = "While getting raw entries for local date, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getRawEntries = function(key_list, start_ts, end_ts, - time_key = "metadata.write_ts", max_entries = undefined, trunc_method = "sample") { - return new Promise(function(resolve, reject) { - var msgFiller = function(message) { - message.key_list = key_list; - message.start_time = start_ts; - message.end_time = end_ts; - message.key_time = time_key; - if (max_entries !== undefined) { - message.max_entries = max_entries; - message.trunc_method = trunc_method; - } - console.log("About to return message "+JSON.stringify(message)); - }; - console.log("getRawEntries: about to get pushGetJSON for the timestamp"); - window.cordova.plugins.BEMServerComm.pushGetJSON("/datastreams/find_entries/timestamp", msgFiller, resolve, reject); - }) - .catch(error => { - error = "While getting raw entries, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getPipelineCompleteTs = function() { - return new Promise(function(resolve, reject) { - console.log("getting pipeline complete timestamp"); - window.cordova.plugins.BEMServerComm.getUserPersonalData("/pipeline/get_complete_ts", resolve, reject); - }) - .catch(error => { - error = "While getting pipeline complete timestamp, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - this.getPipelineRangeTs = function() { - return new Promise(function(resolve, reject) { - console.log("getting pipeline range timestamps"); - window.cordova.plugins.BEMServerComm.getUserPersonalData("/pipeline/get_range_ts", resolve, reject); - }) - .catch(error => { - error = "While getting pipeline range timestamps, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; - - - // host is automatically read from $rootScope.connectUrl, which is set in app.js - this.getAggregateData = function(path, data) { - return new Promise(function(resolve, reject) { - const full_url = $rootScope.connectUrl+"/"+path; - data["aggregate"] = true - - if ($rootScope.aggregateAuth === "no_auth") { - console.log("getting aggregate data without user authentication from " - + full_url +" with arguments "+JSON.stringify(data)); - const options = { - method: 'post', - data: data, - responseType: 'json' - } - cordova.plugin.http.sendRequest(full_url, options, - function(response) { - resolve(response.data); - }, function(error) { - reject(error); - }); - } else { - console.log("getting aggregate data with user authentication from " - + full_url +" with arguments "+JSON.stringify(data)); - var msgFiller = function(message) { - return Object.assign(message, data); - }; - window.cordova.plugins.BEMServerComm.pushGetJSON("/"+path, msgFiller, resolve, reject); - } - }) - .catch(error => { - error = "While getting aggregate data, " + error; - error = processErrorMessages(error); - throw(error); - }); - }; -}) - .service('ReferHelper', function($http) { this.habiticaRegister = function(groupid, successCallback, errorCallback) { @@ -282,7 +22,7 @@ angular.module('emission.services', ['emission.plugin.logger']) //}*/ } }) -.service('UnifiedDataLoader', function($window, CommHelper, Logger) { +.service('UnifiedDataLoader', function($window, Logger) { var combineWithDedup = function(list1, list2) { var combinedList = list1.concat(list2); return combinedList.filter(function(value, i, array) { @@ -364,7 +104,6 @@ angular.module('emission.services', ['emission.plugin.logger']) }) .service('ControlHelper', function($window, $ionicPopup, - CommHelper, Logger) { this.writeFile = function(fileEntry, resultList) { diff --git a/www/js/splash/notifScheduler.js b/www/js/splash/notifScheduler.js index 530439df8..22f8407ee 100644 --- a/www/js/splash/notifScheduler.js +++ b/www/js/splash/notifScheduler.js @@ -9,8 +9,7 @@ angular.module('emission.splash.notifscheduler', ['emission.services', 'emission.plugin.logger']) -.factory('NotificationScheduler', function($http, $window, $ionicPlatform, - CommHelper, Logger) { +.factory('NotificationScheduler', function($http, $window, $ionicPlatform, Logger) { const scheduler = {}; let _config; diff --git a/www/js/splash/pushnotify.js b/www/js/splash/pushnotify.js index f846e41ed..40d859f09 100644 --- a/www/js/splash/pushnotify.js +++ b/www/js/splash/pushnotify.js @@ -20,7 +20,7 @@ angular.module('emission.splash.pushnotify', ['emission.plugin.logger', 'emission.services', 'emission.splash.startprefs']) .factory('PushNotify', function($window, $state, $rootScope, $ionicPlatform, - $ionicPopup, Logger, CommHelper, StartPrefs) { + $ionicPopup, Logger, StartPrefs) { var pushnotify = {}; var push = null; diff --git a/www/js/splash/remotenotify.js b/www/js/splash/remotenotify.js index c11783a2b..3e43b6f9f 100644 --- a/www/js/splash/remotenotify.js +++ b/www/js/splash/remotenotify.js @@ -18,8 +18,7 @@ import { addStatEvent, statKeys } from '../plugin/clientStats'; angular.module('emission.splash.remotenotify', ['emission.plugin.logger', 'emission.splash.startprefs']) -.factory('RemoteNotify', function($http, $window, $ionicPopup, $rootScope, - CommHelper, Logger) { +.factory('RemoteNotify', function($http, $window, $ionicPopup, $rootScope, Logger) { var remoteNotify = {}; remoteNotify.options = "location=yes,clearcache=no,toolbar=yes,hideurlbar=yes"; diff --git a/www/js/splash/storedevicesettings.js b/www/js/splash/storedevicesettings.js index c9c89d5de..d307feaa7 100644 --- a/www/js/splash/storedevicesettings.js +++ b/www/js/splash/storedevicesettings.js @@ -5,7 +5,7 @@ angular.module('emission.splash.storedevicesettings', ['emission.plugin.logger', 'emission.services', 'emission.splash.startprefs']) .factory('StoreDeviceSettings', function($window, $state, $rootScope, $ionicPlatform, - $ionicPopup, Logger, CommHelper, StartPrefs) { + $ionicPopup, Logger, StartPrefs) { var storedevicesettings = {}; From 77fe30a90735db7477df05b7e5009c98a913b319 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Fri, 29 Sep 2023 13:13:20 -0400 Subject: [PATCH 16/25] add commHelper.test.ts -Added one test for `fetchUrlCached` - uses a mocked 'fetch' to simulate retreieving data from a URL. It ensures that the second time this URL is queried, the data comes back faster (because it gets cached in the localStorage after the first call) Note at the bottom explains why other functions were not tested. --- www/__tests__/commHelper.test.ts | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 www/__tests__/commHelper.test.ts diff --git a/www/__tests__/commHelper.test.ts b/www/__tests__/commHelper.test.ts new file mode 100644 index 000000000..6143381e3 --- /dev/null +++ b/www/__tests__/commHelper.test.ts @@ -0,0 +1,41 @@ +import { mockLogger } from '../__mocks__/globalMocks'; +import { fetchUrlCached } from '../js/commHelper'; + +mockLogger(); + +// mock for JavaScript 'fetch' +// we emulate a 100ms delay when i) fetching data and ii) parsing it as text +global.fetch = (url: string) => new Promise((rs, rj) => { + setTimeout(() => rs({ + text: () => new Promise((rs, rj) => { + setTimeout(() => rs('mock data for ' + url), 100); + }) + })); +}) as any; + +it('fetches text from a URL and caches it so the next call is faster', async () => { + const tsBeforeCalls = Date.now(); + const text1 = await fetchUrlCached('https://raw.githubusercontent.com/e-mission/e-mission-phone/master/README.md'); + const tsBetweenCalls = Date.now(); + const text2 = await fetchUrlCached('https://raw.githubusercontent.com/e-mission/e-mission-phone/master/README.md'); + const tsAfterCalls = Date.now(); + expect(text1).toEqual(expect.stringContaining('mock data')); + expect(text2).toEqual(expect.stringContaining('mock data')); + expect(tsAfterCalls - tsBetweenCalls).toBeLessThan(tsBetweenCalls - tsBeforeCalls); +}); + +/* The following functions from commHelper.ts are not tested because they are just wrappers + around the native functions in BEMServerComm. + If we wanted to test them, we would need to mock the native functions in BEMServerComm, but + this would be of limited value. It would be better to test the native functions directly. + + * - getRawEntries + * - getPipelineRangeTs + * - getPipelineCompleteTs + * - getMetrics + * - getAggregateData + * - registerUser + * - updateUser + * - getUser + +*/ From a5fcbfe1be3b740cbd849d0722c2f1de1e38269e Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 11 Oct 2023 16:42:12 -0400 Subject: [PATCH 17/25] remove backwards-compat munge in storage.ts per https://github.com/e-mission/e-mission-phone/pull/1040/files#r1351511049, removes the backwards compat to munge when filling in native values from local storage. Also removes the backwards-compat comment, replacing it with other comments describing how we fill in missing local or native values --- www/js/plugin/storage.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts index 87be6de9b..68380fe86 100644 --- a/www/js/plugin/storage.ts +++ b/www/js/plugin/storage.ts @@ -43,7 +43,7 @@ const localStorageGet = (key: string) => { If a value is present in both, but they are different, it copies the native value to local storage and returns it. */ function getUnifiedValue(key) { - let ls_stored_val = localStorageGet(key); + const ls_stored_val = localStorageGet(key); return window['cordova'].plugins.BEMUserCache.getLocalStorage(key, false).then((uc_stored_val) => { logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}.`); @@ -56,6 +56,7 @@ function getUnifiedValue(key) { } else { // the values are different if (ls_stored_val == null) { + // local value is missing, fill it in from native console.assert(uc_stored_val != null, "uc_stored_val should be non-null"); logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}. @@ -63,14 +64,8 @@ function getUnifiedValue(key) { localStorageSet(key, uc_stored_val); return uc_stored_val; } else if (uc_stored_val == null) { + // native value is missing, fill it in from local console.assert(ls_stored_val != null); - /* - * Backwards compatibility ONLY. Right after the first - * update to this version, we may have a local value that - * is not a JSON object. In that case, we want to munge it - * before storage. Remove this after a few releases. - */ - ls_stored_val = mungeValue(key, ls_stored_val); displayErrorMsg(`Local ${key} found, native ${key} missing, writing ${key} to native`); logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}. @@ -80,6 +75,7 @@ function getUnifiedValue(key) { return ls_stored_val; }); } + // both values are present, but they are different console.assert(ls_stored_val != null && uc_stored_val != null, "ls_stored_val =" + JSON.stringify(ls_stored_val) + "uc_stored_val =" + JSON.stringify(uc_stored_val)); From 40ce2293b344dee2ed4fdca8ebffef53eac8dc7a Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 11 Oct 2023 16:49:27 -0400 Subject: [PATCH 18/25] don't displayErrorMsg in storage.ts A user-facing popup here is likely to just annoy or confuse users. Let's set these log statements at the "WARN" level so it is more visible than other log statements if we do need to debug it, but not intrusive or detrimental to UX --- www/js/plugin/storage.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts index 68380fe86..3fc67b616 100644 --- a/www/js/plugin/storage.ts +++ b/www/js/plugin/storage.ts @@ -1,6 +1,6 @@ import { getAngularService } from "../angular-react-helper"; import { addStatReading, statKeys } from "./clientStats"; -import { displayErrorMsg, logDebug } from "./logger"; +import { logDebug, logWarn } from "./logger"; const mungeValue = (key, value) => { let store_val = value; @@ -58,7 +58,7 @@ function getUnifiedValue(key) { if (ls_stored_val == null) { // local value is missing, fill it in from native console.assert(uc_stored_val != null, "uc_stored_val should be non-null"); - logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + logWarn(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}. copying native ${key} to local...`); localStorageSet(key, uc_stored_val); @@ -66,8 +66,7 @@ function getUnifiedValue(key) { } else if (uc_stored_val == null) { // native value is missing, fill it in from local console.assert(ls_stored_val != null); - displayErrorMsg(`Local ${key} found, native ${key} missing, writing ${key} to native`); - logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + logWarn(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}. copying local ${key} to native...`); return window['cordova'].plugins.BEMUserCache.putLocalStorage(key, ls_stored_val).then(() => { @@ -79,9 +78,7 @@ function getUnifiedValue(key) { console.assert(ls_stored_val != null && uc_stored_val != null, "ls_stored_val =" + JSON.stringify(ls_stored_val) + "uc_stored_val =" + JSON.stringify(uc_stored_val)); - displayErrorMsg(`Local ${key} found, native ${key} found, but different, - writing ${key} to local`); - logDebug(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, + logWarn(`for key ${key}, uc_stored_val = ${JSON.stringify(uc_stored_val)}, ls_stored_val = ${JSON.stringify(ls_stored_val)}. copying native ${key} to local...`); localStorageSet(key, uc_stored_val); From 37ac06581f3bb364a3c4fb54849e19f90015a853 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 11 Oct 2023 16:51:29 -0400 Subject: [PATCH 19/25] remove unneeded comment It is fairly self-explanatory that {key: value} was the chosen approach --- www/js/plugin/storage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/www/js/plugin/storage.ts b/www/js/plugin/storage.ts index 3fc67b616..59e535b6e 100644 --- a/www/js/plugin/storage.ts +++ b/www/js/plugin/storage.ts @@ -5,7 +5,6 @@ import { logDebug, logWarn } from "./logger"; const mungeValue = (key, value) => { let store_val = value; if (typeof value != "object") { - // Should this be {"value": value} or {key: value}? store_val = {}; store_val[key] = value; } From cfd782991f59ea9915dcd2665e880e9ae6a4b1e3 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 11 Oct 2023 17:04:55 -0400 Subject: [PATCH 20/25] show error if 'db' not defined in clientStats.ts Per https://github.com/e-mission/e-mission-phone/pull/1040/files#r1351477569, we'll show an error message here if the BEMUserCache 'db' is not defined. We'll also ensure that if the Logger plugin is undefined, we do not try to call it as this would cause an error. --- www/js/plugin/clientStats.ts | 5 +++++ www/js/plugin/logger.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/www/js/plugin/clientStats.ts b/www/js/plugin/clientStats.ts index 1e06208eb..cefaf8f22 100644 --- a/www/js/plugin/clientStats.ts +++ b/www/js/plugin/clientStats.ts @@ -1,3 +1,5 @@ +import { displayErrorMsg } from "./logger"; + const CLIENT_TIME = "stats/client_time"; const CLIENT_ERROR = "stats/client_error"; const CLIENT_NAV_EVENT = "stats/client_nav_event"; @@ -39,16 +41,19 @@ export const addStatReading = async (name: string, reading: any) => { const db = window['cordova']?.plugins?.BEMUserCache; const event = await getStatsEvent(name, reading); if (db) return db.putMessage(CLIENT_TIME, event); + displayErrorMsg("addStatReading: db is not defined"); } export const addStatEvent = async (name: string) => { const db = window['cordova']?.plugins?.BEMUserCache; const event = await getStatsEvent(name, null); if (db) return db.putMessage(CLIENT_NAV_EVENT, event); + displayErrorMsg("addStatEvent: db is not defined"); } export const addStatError = async (name: string, errorStr: string) => { const db = window['cordova']?.plugins?.BEMUserCache; const event = await getStatsEvent(name, errorStr); if (db) return db.putMessage(CLIENT_ERROR, event); + displayErrorMsg("addStatError: db is not defined"); } diff --git a/www/js/plugin/logger.ts b/www/js/plugin/logger.ts index c4e476de1..d127f5549 100644 --- a/www/js/plugin/logger.ts +++ b/www/js/plugin/logger.ts @@ -46,5 +46,5 @@ export function displayErrorMsg(errorMsg: string, title?: string) { const displayMsg = `━━━━\n${title}\n━━━━\n` + errorMsg; window.alert(displayMsg); console.error(displayMsg); - window['Logger'].log(window['Logger'].LEVEL_ERROR, displayMsg); + window['Logger']?.log(window['Logger'].LEVEL_ERROR, displayMsg); } From ea2b8c59ba0d4c4cfdeac69c74712590d37345ed Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 12 Oct 2023 11:28:41 -0400 Subject: [PATCH 21/25] re-implement functions of commHelper A couple of the functions that were excluded from the rewrite were requested to be added back: https://github.com/e-mission/e-mission-phone/pull/1040#discussion_r1350931942 The logic is the same, but with modernized syntax, and using Luxon instead of Moment to deal with timestamps --- www/js/commHelper.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/www/js/commHelper.ts b/www/js/commHelper.ts index d0042b6ab..85281694f 100644 --- a/www/js/commHelper.ts +++ b/www/js/commHelper.ts @@ -1,3 +1,4 @@ +import { DateTime } from "luxon"; import { logDebug } from "./plugin/logger"; /** @@ -49,6 +50,29 @@ export function getRawEntries(key_list, start_ts, end_ts, time_key = "metadata.w }); } +// time_key is typically metadata.write_ts or data.ts +export function getRawEntriesForLocalDate(key_list, start_ts, end_ts, time_key = "metadata.write_ts", + max_entries = undefined, trunc_method = "sample") { + return new Promise((rs, rj) => { + const msgFiller = (message) => { + message.key_list = key_list; + message.from_local_date = DateTime.fromSeconds(start_ts).toObject(); + message.to_local_date = DateTime.fromSeconds(end_ts).toObject(); + message.key_local_date = time_key; + if (max_entries !== undefined) { + message.max_entries = max_entries; + message.trunc_method = trunc_method; + } + logDebug("About to return message " + JSON.stringify(message)); + }; + logDebug("getRawEntries: about to get pushGetJSON for the timestamp"); + window['cordova'].plugins.BEMServerComm.pushGetJSON("/datastreams/find_entries/local_date", msgFiller, rs, rj); + }).catch(error => { + error = "While getting raw entries for local date, " + error; + throw (error); + }); +}; + export function getPipelineRangeTs() { return new Promise((rs, rj) => { logDebug("getting pipeline range timestamps"); @@ -141,3 +165,22 @@ export function getUser() { throw(error); }); } + +export function putOne(key, data) { + const nowTs = DateTime.now().toUnixInteger(); + const metadata = { + write_ts: nowTs, + read_ts: nowTs, + time_zone: DateTime.local().zoneName, + type: "message", + key: key, + platform: window['device'].platform, + }; + const entryToPut = { metadata, data }; + return new Promise((rs, rj) => { + window['cordova'].plugins.BEMServerComm.postUserPersonalData("/usercache/putone", "the_entry", entryToPut, rs, rj); + }).catch(error => { + error = "While putting one entry, " + error; + throw(error); + }); +}; From 641e8aa212b6d9558e5d98e7ec5e312636f927f3 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 12 Oct 2023 13:15:23 -0400 Subject: [PATCH 22/25] don't "processErrorMessages" in commHelper Appending "user-friendy" descriptions to error popups is already handled in logger.ts (see `displayErrorMsg`), so it is unnecessary in this file. --- www/js/commHelper.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/www/js/commHelper.ts b/www/js/commHelper.ts index 85281694f..259677090 100644 --- a/www/js/commHelper.ts +++ b/www/js/commHelper.ts @@ -19,14 +19,6 @@ export async function fetchUrlCached(url) { return text; } -function processErrorMessages(errorMsg) { - if (errorMsg.includes("403")) { - errorMsg = "Error: OPcode does not exist on the server. " + errorMsg; - console.error("Error 403 found. " + errorMsg); - } - return errorMsg; -} - export function getRawEntries(key_list, start_ts, end_ts, time_key = "metadata.write_ts", max_entries = undefined, trunc_method = "sample") { return new Promise((rs, rj) => { @@ -45,7 +37,6 @@ export function getRawEntries(key_list, start_ts, end_ts, time_key = "metadata.w window['cordova'].plugins.BEMServerComm.pushGetJSON("/datastreams/find_entries/timestamp", msgFiller, rs, rj); }).catch(error => { error = `While getting raw entries, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -79,7 +70,6 @@ export function getPipelineRangeTs() { window['cordova'].plugins.BEMServerComm.getUserPersonalData("/pipeline/get_range_ts", rs, rj); }).catch(error => { error = `While getting pipeline range timestamps, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -90,7 +80,6 @@ export function getPipelineCompleteTs() { window['cordova'].plugins.BEMServerComm.getUserPersonalData("/pipeline/get_complete_ts", rs, rj); }).catch(error => { error = `While getting pipeline complete timestamp, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -105,7 +94,6 @@ export function getMetrics(timeType: 'timestamp'|'local_date', metricsQuery) { window['cordova'].plugins.BEMServerComm.pushGetJSON(`/result/metrics/${timeType}`, msgFiller, rs, rj); }).catch(error => { error = `While getting metrics, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -137,7 +125,6 @@ export function getAggregateData(path: string, data: any) { } }).catch(error => { error = `While getting aggregate data, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -151,7 +138,6 @@ export function updateUser(updateDoc) { window['cordova'].plugins.BEMServerComm.postUserPersonalData("/profile/update", "update_doc", updateDoc, rs, rj); }).catch(error => { error = `While updating user, ${error}`; - error = processErrorMessages(error); throw(error); }); } @@ -161,7 +147,6 @@ export function getUser() { window['cordova'].plugins.BEMServerComm.getUserPersonalData("/profile/get", rs, rj); }).catch(error => { error = `While getting user, ${error}`; - error = processErrorMessages(error); throw(error); }); } From 03dc94e0fa3c6b943c2bc28bf69f72e1df99fc01 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 12 Oct 2023 13:27:12 -0400 Subject: [PATCH 23/25] update comment in commHelper.test.ts - Add the 2 functions to the list that were re-implemented after the initial rewrite - More accurately describe how we would ideally test server comm --- www/__tests__/commHelper.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/www/__tests__/commHelper.test.ts b/www/__tests__/commHelper.test.ts index 6143381e3..2e2dfc6af 100644 --- a/www/__tests__/commHelper.test.ts +++ b/www/__tests__/commHelper.test.ts @@ -26,10 +26,11 @@ it('fetches text from a URL and caches it so the next call is faster', async () /* The following functions from commHelper.ts are not tested because they are just wrappers around the native functions in BEMServerComm. - If we wanted to test them, we would need to mock the native functions in BEMServerComm, but - this would be of limited value. It would be better to test the native functions directly. + If we wanted to test them, we would need to mock the native functions in BEMServerComm. + It would be better to do integration tests that actually call the native functions. * - getRawEntries + * - getRawEntriesForLocalDate * - getPipelineRangeTs * - getPipelineCompleteTs * - getMetrics @@ -37,5 +38,5 @@ it('fetches text from a URL and caches it so the next call is faster', async () * - registerUser * - updateUser * - getUser - + * - putOne */ From acb36aad8434a6d1ff91bf33b19f9ec86a8509aa Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 12 Oct 2023 13:57:18 -0400 Subject: [PATCH 24/25] cordovaMocks: use cordova-ios version from json --- www/__mocks__/cordovaMocks.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/www/__mocks__/cordovaMocks.ts b/www/__mocks__/cordovaMocks.ts index 44c21677c..4a9189ecd 100644 --- a/www/__mocks__/cordovaMocks.ts +++ b/www/__mocks__/cordovaMocks.ts @@ -1,7 +1,9 @@ +import packageJsonBuild from '../../package.cordovabuild.json'; + export const mockCordova = () => { window['cordova'] ||= {}; window['cordova'].platformId ||= 'ios'; - window['cordova'].platformVersion ||= '6.2.0'; + window['cordova'].platformVersion ||= packageJsonBuild.dependencies['cordova-ios']; window['cordova'].plugins ||= {}; } From e546d737a2a67e7e64660dbcb6b2fd4b79e4dfbf Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Thu, 12 Oct 2023 14:05:59 -0400 Subject: [PATCH 25/25] commHelper: promisify 'registerUser' Per https://github.com/e-mission/e-mission-phone/pull/1040#discussion_r1350911732 --- www/js/commHelper.ts | 9 +++++++-- www/js/onboarding/SaveQrPage.tsx | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/www/js/commHelper.ts b/www/js/commHelper.ts index 259677090..b9584a044 100644 --- a/www/js/commHelper.ts +++ b/www/js/commHelper.ts @@ -129,8 +129,13 @@ export function getAggregateData(path: string, data: any) { }); } -export function registerUser(successCallback, errorCallback) { - window['cordova'].plugins.BEMServerComm.getUserPersonalData("/profile/create", successCallback, errorCallback); +export function registerUser() { + return new Promise((rs, rj) => { + window['cordova'].plugins.BEMServerComm.getUserPersonalData("/profile/create", rs, rj); + }).catch(error => { + error = `While registering user, ${error}`; + throw(error); + }); } export function updateUser(updateDoc) { diff --git a/www/js/onboarding/SaveQrPage.tsx b/www/js/onboarding/SaveQrPage.tsx index d8b555f14..6fcf06e8c 100644 --- a/www/js/onboarding/SaveQrPage.tsx +++ b/www/js/onboarding/SaveQrPage.tsx @@ -37,10 +37,10 @@ const SaveQrPage = ({ }) => { const EXPECTED_METHOD = "prompted-auth"; const dbStorageObject = {"token": token}; return storageSet(EXPECTED_METHOD, dbStorageObject).then((r) => { - registerUser((successResult) => { + registerUser().then((r) => { refreshOnboardingState(); - }, function(errorResult) { - displayError(errorResult, "User registration error"); + }).catch((e) => { + displayError(e, "User registration error"); }); }).catch((e) => { displayError(e, "Sign in error");