From 6086cc7226a4ad0ff2b485742140e5d49a739ae4 Mon Sep 17 00:00:00 2001 From: Shubham Goyal Date: Fri, 1 Dec 2023 10:18:41 -0500 Subject: [PATCH 1/5] Switch to async factory when cache and index is available --- .../util/screen/EntityScreenHelper.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java index 69c078be5..94eb296ed 100644 --- a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java +++ b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java @@ -1,8 +1,11 @@ package org.commcare.util.screen; +import org.commcare.cases.entity.AsyncEntity; +import org.commcare.cases.entity.AsyncNodeEntityFactory; import org.commcare.cases.entity.Entity; import org.commcare.cases.entity.EntitySortNotificationInterface; import org.commcare.cases.entity.EntitySorter; +import org.commcare.cases.entity.EntityStorageCache; import org.commcare.cases.entity.EntityStringFilterer; import org.commcare.cases.entity.NodeEntityFactory; import org.commcare.suite.model.Detail; @@ -13,6 +16,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -32,7 +36,48 @@ public class EntityScreenHelper { */ public static List> initEntities(EvaluationContext context, Detail detail, EntityScreenContext entityScreenContext, TreeReference[] entitiesRefs) { - NodeEntityFactory nodeEntityFactory = new NodeEntityFactory(detail, context); + NodeEntityFactory nodeEntityFactory; + if (detail.useAsyncStrategy()) { + nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, new EntityStorageCache() { + @Override + public boolean lockCache() { + return true; + } + + @Override + public void releaseCache() { + + } + + @Override + public String getCacheKey(String detailId, String detailFieldIndex) { + return detailId + "_" + detailFieldIndex; + } + + @Override + public String retrieveCacheValue(String cacheIndex, String cacheKey) { + return null; + } + + @Override + public void cache(String cacheIndex, String cacheKey, String data) { + + } + + @Override + public int getSortFieldIdFromCacheKey(String detailId, String cacheKey) { + return -1; + } + + @Override + public void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, + Detail detail) { + + } + }); + } else { + nodeEntityFactory = new NodeEntityFactory(detail, context); + } List> entities = new ArrayList<>(); for (TreeReference reference : entitiesRefs) { entities.add(nodeEntityFactory.getEntity(reference)); From 8daf3d31d131f4352fffe42c1c9e851aa95a7536 Mon Sep 17 00:00:00 2001 From: Shubham Goyal Date: Fri, 1 Dec 2023 11:21:58 -0500 Subject: [PATCH 2/5] Adds grouping to async Entity --- .../org/commcare/cases/entity/AsyncEntity.java | 16 +++++++++++++++- .../cases/entity/AsyncNodeEntityFactory.java | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/commcare/cases/entity/AsyncEntity.java b/src/main/java/org/commcare/cases/entity/AsyncEntity.java index fbe5b43e1..626474acf 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncEntity.java +++ b/src/main/java/org/commcare/cases/entity/AsyncEntity.java @@ -3,6 +3,7 @@ import org.commcare.cases.util.StringUtils; import org.commcare.suite.model.DetailField; +import org.commcare.suite.model.DetailGroup; import org.commcare.suite.model.Text; import org.javarosa.core.model.condition.EvaluationContext; import org.javarosa.core.model.instance.TreeReference; @@ -15,6 +16,8 @@ import java.util.Enumeration; import java.util.Hashtable; +import javax.annotation.Nullable; + /** * An AsyncEntity is an entity reference which is capable of building its * values (evaluating all Text elements/background data elements) lazily @@ -38,6 +41,7 @@ public class AsyncEntity extends Entity { private final String[][] sortDataPieces; private final EvaluationContext context; private final Hashtable mVariableDeclarations; + private final DetailGroup mDetailGroup; private boolean mVariableContextLoaded = false; private final String mCacheIndex; private final String mDetailId; @@ -60,7 +64,7 @@ public class AsyncEntity extends Entity { public AsyncEntity(DetailField[] fields, EvaluationContext ec, TreeReference t, Hashtable variables, EntityStorageCache cache, String cacheIndex, String detailId, - String extraKey) { + String extraKey, DetailGroup detailGroup) { super(t, extraKey); this.fields = fields; @@ -77,6 +81,7 @@ public AsyncEntity(DetailField[] fields, EvaluationContext ec, this.mCacheIndex = cacheIndex; this.mDetailId = detailId; + this.mDetailGroup = detailGroup; } private void loadVariableContext() { @@ -233,4 +238,13 @@ private static String[] breakUpField(String input) { return input.split("\\s+"); } } + + @Nullable + @Override + public String getGroupKey() { + if (mDetailGroup != null) { + return (String)mDetailGroup.getFunction().eval(context); + } + return null; + } } diff --git a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java index e766b0619..9d06967a1 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java +++ b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java @@ -54,7 +54,7 @@ public Entity getEntity(TreeReference data) { String entityKey = loadCalloutDataMapKey(nodeContext); AsyncEntity entity = new AsyncEntity(detail.getFields(), nodeContext, data, mVariableDeclarations, - mEntityCache, mCacheIndex, detail.getId(), entityKey); + mEntityCache, mCacheIndex, detail.getId(), entityKey, detail.getGroup()); if (mCacheIndex != null) { mEntitySet.put(mCacheIndex, entity); From 89625e8ab0b653603560e88ab45d53903644a996 Mon Sep 17 00:00:00 2001 From: Shubham Goyal Date: Fri, 1 Dec 2023 16:33:35 -0500 Subject: [PATCH 3/5] makes entity cache nullable for async entity framework and pass in a null cache for web apps --- .../util/screen/EntityScreenHelper.java | 38 +-------- .../commcare/cases/entity/AsyncEntity.java | 80 +++++++++++-------- .../cases/entity/AsyncNodeEntityFactory.java | 8 +- 3 files changed, 53 insertions(+), 73 deletions(-) diff --git a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java index 94eb296ed..801e463ca 100644 --- a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java +++ b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java @@ -38,43 +38,7 @@ public static List> initEntities(EvaluationContext context EntityScreenContext entityScreenContext, TreeReference[] entitiesRefs) { NodeEntityFactory nodeEntityFactory; if (detail.useAsyncStrategy()) { - nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, new EntityStorageCache() { - @Override - public boolean lockCache() { - return true; - } - - @Override - public void releaseCache() { - - } - - @Override - public String getCacheKey(String detailId, String detailFieldIndex) { - return detailId + "_" + detailFieldIndex; - } - - @Override - public String retrieveCacheValue(String cacheIndex, String cacheKey) { - return null; - } - - @Override - public void cache(String cacheIndex, String cacheKey, String data) { - - } - - @Override - public int getSortFieldIdFromCacheKey(String detailId, String cacheKey) { - return -1; - } - - @Override - public void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, - Detail detail) { - - } - }); + nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, null); } else { nodeEntityFactory = new NodeEntityFactory(detail, context); } diff --git a/src/main/java/org/commcare/cases/entity/AsyncEntity.java b/src/main/java/org/commcare/cases/entity/AsyncEntity.java index 626474acf..99d496165 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncEntity.java +++ b/src/main/java/org/commcare/cases/entity/AsyncEntity.java @@ -46,6 +46,7 @@ public class AsyncEntity extends Entity { private final String mCacheIndex; private final String mDetailId; + @Nullable private final EntityStorageCache mEntityStorageCache; /* @@ -63,7 +64,7 @@ public class AsyncEntity extends Entity { public AsyncEntity(DetailField[] fields, EvaluationContext ec, TreeReference t, Hashtable variables, - EntityStorageCache cache, String cacheIndex, String detailId, + @Nullable EntityStorageCache cache, String cacheIndex, String detailId, String extraKey, DetailGroup detailGroup) { super(t, extraKey); @@ -126,47 +127,55 @@ public String getNormalizedField(int i) { @Override public String getSortField(int i) { - if (mEntityStorageCache.lockCache()) { - //get our second lock. - synchronized (mAsyncLock) { - if (sortData[i] == null) { - // sort data not in search field cache; load and store it - Text sortText = fields[i].getSort(); - if (sortText == null) { - mEntityStorageCache.releaseCache(); - return null; - } + try { + if (mEntityStorageCache == null || mEntityStorageCache.lockCache()) { + //get our second lock. + synchronized (mAsyncLock) { + if (sortData[i] == null) { + // sort data not in search field cache; load and store it + Text sortText = fields[i].getSort(); + if (sortText == null) { + return null; + } - String cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i)); + String cacheKey = null; + if (mEntityStorageCache != null) { + cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i)); - if (mCacheIndex != null) { - //Check the cache! - String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); - if (value != null) { - this.setSortData(i, value); - mEntityStorageCache.releaseCache(); - return sortData[i]; + if (mCacheIndex != null) { + //Check the cache! + String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); + if (value != null) { + this.setSortData(i, value); + return sortData[i]; + } + } } - } - loadVariableContext(); - try { - sortText = fields[i].getSort(); - if (sortText == null) { - this.setSortData(i, getFieldString(i)); - } else { - this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); + loadVariableContext(); + try { + sortText = fields[i].getSort(); + if (sortText == null) { + this.setSortData(i, getFieldString(i)); + } else { + this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); + } + if (mEntityStorageCache != null) { + mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); + } + } catch (XPathException xpe) { + Logger.exception("Error while evaluating sort field", xpe); + xpe.printStackTrace(); + sortData[i] = ""; } - - mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); - } catch (XPathException xpe) { - Logger.exception("Error while evaluating sort field", xpe); - xpe.printStackTrace(); - sortData[i] = ""; } + + return sortData[i]; } + } + } finally { + if (mEntityStorageCache != null) { mEntityStorageCache.releaseCache(); - return sortData[i]; } } return null; @@ -223,6 +232,9 @@ private void setSortData(int i, String val) { } public void setSortData(String cacheKey, String val) { + if (mEntityStorageCache == null) { + throw new IllegalStateException("No entity cache defined"); + } int sortIndex = mEntityStorageCache.getSortFieldIdFromCacheKey(mDetailId, cacheKey); if (sortIndex != -1) { setSortData(sortIndex, val); diff --git a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java index 9d06967a1..6f2821ed9 100755 --- a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java +++ b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java @@ -11,6 +11,8 @@ import java.util.Hashtable; import java.util.List; +import javax.annotation.Nullable; + /** * @author ctsims */ @@ -19,6 +21,7 @@ public class AsyncNodeEntityFactory extends NodeEntityFactory { private final OrderedHashtable mVariableDeclarations; private final Hashtable mEntitySet = new Hashtable<>(); + @Nullable private final EntityStorageCache mEntityCache; private CacheHost mCacheHost = null; @@ -29,7 +32,8 @@ public class AsyncNodeEntityFactory extends NodeEntityFactory { // Don't show entity list until we primeCache and caches all fields private final boolean isBlockingAsyncMode; - public AsyncNodeEntityFactory(Detail d, EvaluationContext ec, EntityStorageCache entityStorageCache) { + public AsyncNodeEntityFactory(Detail d, EvaluationContext ec, + @Nullable EntityStorageCache entityStorageCache) { super(d, ec); mVariableDeclarations = detail.getVariableDeclarations(); @@ -77,7 +81,7 @@ protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec, * Note that the cache is lazily built upon first case list search. */ private void primeCache() { - if (mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) { + if (mEntityCache == null || mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) { return; } From bbc3d723a7a31d60a2b662390c4041aab7a36135 Mon Sep 17 00:00:00 2001 From: Shubham Goyal Date: Mon, 4 Dec 2023 12:31:09 -0500 Subject: [PATCH 4/5] adds lazy_loading in detail config --- src/main/java/org/commcare/suite/model/Detail.java | 13 ++++++++++++- src/main/java/org/commcare/xml/DetailParser.java | 3 ++- .../backend/suite/model/test/AppStructureTests.java | 6 ++++++ src/test/resources/app_structure/suite.xml | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/commcare/suite/model/Detail.java b/src/main/java/org/commcare/suite/model/Detail.java index 8aec0bc98..6f9081ac0 100755 --- a/src/main/java/org/commcare/suite/model/Detail.java +++ b/src/main/java/org/commcare/suite/model/Detail.java @@ -104,6 +104,9 @@ public class Detail implements Externalizable { // equal to its width, rather than being computed independently private boolean useUniformUnitsInCaseTile; + // Loads detail fields lazily when required + private boolean lazyLoading; + private DetailGroup group; // ENDREGION @@ -119,7 +122,8 @@ public Detail(String id, DisplayUnit title, Text noItemsText, String nodeset, Ve Vector fieldsVector, OrderedHashtable variables, Vector actions, Callout callout, String fitAcross, String uniformUnitsString, String forceLandscape, String focusFunction, - String printPathProvided, String relevancy, Global global, DetailGroup group) { + String printPathProvided, String relevancy, Global global, DetailGroup group, + boolean lazyLoading) { if (detailsVector.size() > 0 && fieldsVector.size() > 0) { throw new IllegalArgumentException("A detail may contain either sub-details or fields, but not both."); @@ -169,6 +173,7 @@ public Detail(String id, DisplayUnit title, Text noItemsText, String nodeset, Ve } this.global = global; this.group = group; + this.lazyLoading = lazyLoading; } /** @@ -211,6 +216,10 @@ public Detail[] getDetails() { return details; } + public boolean isLazyLoading() { + return lazyLoading; + } + /** * Given a detail, return an array of details that will contain either * - all child details @@ -274,6 +283,7 @@ public void readExternal(DataInputStream in, PrototypeFactory pf) throws IOExcep parsedRelevancyExpression = (XPathExpression)ExtUtil.read(in, new ExtWrapNullable(new ExtWrapTagged()), pf); global = (Global)ExtUtil.read(in, new ExtWrapNullable(new ExtWrapTagged()), pf); group = (DetailGroup) ExtUtil.read(in, new ExtWrapNullable(DetailGroup.class), pf); + lazyLoading = ExtUtil.readBool(in); } @Override @@ -296,6 +306,7 @@ public void writeExternal(DataOutputStream out) throws IOException { parsedRelevancyExpression == null ? null : new ExtWrapTagged(parsedRelevancyExpression))); ExtUtil.write(out, new ExtWrapNullable(global == null ? null : new ExtWrapTagged(global))); ExtUtil.write(out, new ExtWrapNullable(group)); + ExtUtil.writeBool(out, lazyLoading); } public OrderedHashtable getVariableDeclarations() { diff --git a/src/main/java/org/commcare/xml/DetailParser.java b/src/main/java/org/commcare/xml/DetailParser.java index f72da3f34..396c70f56 100755 --- a/src/main/java/org/commcare/xml/DetailParser.java +++ b/src/main/java/org/commcare/xml/DetailParser.java @@ -39,6 +39,7 @@ public Detail parse() throws InvalidStructureException, IOException, XmlPullPars String forceLandscapeView = parser.getAttributeValue(null, "force-landscape"); String printTemplatePath = parser.getAttributeValue(null, "print-template"); String relevancy = parser.getAttributeValue(null, "relevant"); + boolean isLazyLoading = Boolean.parseBoolean(parser.getAttributeValue(null, "lazy_loading")); // First fetch the title getNextTagInBlock("detail"); @@ -131,7 +132,7 @@ public Detail parse() throws InvalidStructureException, IOException, XmlPullPars return new Detail(id, title, noItemsText, nodeset, subdetails, fields, variables, actions, callout, fitAcross, useUniformUnits, forceLandscapeView, focusFunction, printTemplatePath, - relevancy, global, detailGroup); + relevancy, global, detailGroup, isLazyLoading); } protected DetailParser getDetailParser() { diff --git a/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java b/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java index 46ab8193d..bdae24b26 100644 --- a/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java +++ b/src/test/java/org/commcare/backend/suite/model/test/AppStructureTests.java @@ -234,6 +234,12 @@ public void testDetailWithBorder() { assertTrue(field1.getShowBorder()); } + @Test + public void testDetailWithLazyLoadingSet() { + Detail detail = mApp.getSession().getPlatform().getDetail("m0_case_short"); + assertTrue(detail.isLazyLoading()); + } + @Test public void testDefaultEndpointRelevancy_shouldBeTrue() { Endpoint endpoint = mApp.getSession().getPlatform().getEndpoint("endpoint_with_no_relevancy"); diff --git a/src/test/resources/app_structure/suite.xml b/src/test/resources/app_structure/suite.xml index 8134c36e3..63a4c4f1c 100644 --- a/src/test/resources/app_structure/suite.xml +++ b/src/test/resources/app_structure/suite.xml @@ -11,7 +11,7 @@ - + <text>Case List</text> From cfd5a504248b3551bf537aec741f36506d103895 Mon Sep 17 00:00:00 2001 From: Shubham Goyal Date: Mon, 4 Dec 2023 12:43:43 -0500 Subject: [PATCH 5/5] switch to async entity factory if lazy loading is true --- src/cli/java/org/commcare/util/screen/EntityScreenHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java index 801e463ca..1b2df52bc 100644 --- a/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java +++ b/src/cli/java/org/commcare/util/screen/EntityScreenHelper.java @@ -37,7 +37,7 @@ public class EntityScreenHelper { public static List> initEntities(EvaluationContext context, Detail detail, EntityScreenContext entityScreenContext, TreeReference[] entitiesRefs) { NodeEntityFactory nodeEntityFactory; - if (detail.useAsyncStrategy()) { + if (detail.isLazyLoading()) { nodeEntityFactory = new AsyncNodeEntityFactory(detail, context, null); } else { nodeEntityFactory = new NodeEntityFactory(detail, context);