diff --git a/app/res/layout/entity_select_layout.xml b/app/res/layout/entity_select_layout.xml index f59231a47f..307cb99d0f 100644 --- a/app/res/layout/entity_select_layout.xml +++ b/app/res/layout/entity_select_layout.xml @@ -53,6 +53,7 @@ android:visibility="gone"/> @@ -63,8 +64,18 @@ android:layout_width="wrap_content" android:layout_height="wrap_content" android:layout_centerInParent="true"/> + + + + notification-channel-push-notifications Required CommCare App is not installed on device Audio Recording Notification + Initializing list… + Processing %1s out of %2s + Loading Cache… + Calculating %1s out of %2s + Finalizing list… diff --git a/app/src/org/commcare/CommCareApplication.java b/app/src/org/commcare/CommCareApplication.java index e421ca9fc9..7ef93bb27f 100644 --- a/app/src/org/commcare/CommCareApplication.java +++ b/app/src/org/commcare/CommCareApplication.java @@ -97,6 +97,8 @@ import org.commcare.tasks.DataPullTask; import org.commcare.tasks.DeleteLogs; import org.commcare.tasks.LogSubmissionTask; +import org.commcare.tasks.PrimeEntityCache; +import org.commcare.tasks.PrimeEntityCacheHelper; import org.commcare.tasks.PurgeStaleArchivedFormsTask; import org.commcare.tasks.templates.ManagedAsyncTask; import org.commcare.update.UpdateHelper; @@ -387,6 +389,7 @@ protected void cancelWorkManagerTasks() { if (currentApp != null) { WorkManager.getInstance(this).cancelUniqueWork( FormSubmissionHelper.getFormSubmissionRequestName(currentApp.getUniqueId())); + PrimeEntityCacheHelper.cancelWork(); } } @@ -796,7 +799,7 @@ public void onServiceConnected(ComponentName className, IBinder service) { purgeLogs(); cleanRawMedia(); - + PrimeEntityCacheHelper.schedulePrimeEntityCacheWorker(); } TimedStatsTracker.registerStartSession(); diff --git a/app/src/org/commcare/activities/EntitySelectActivity.java b/app/src/org/commcare/activities/EntitySelectActivity.java index 7c4fec9b0a..60506a9408 100755 --- a/app/src/org/commcare/activities/EntitySelectActivity.java +++ b/app/src/org/commcare/activities/EntitySelectActivity.java @@ -22,7 +22,6 @@ import android.widget.TextView; import android.widget.Toast; -import androidx.appcompat.app.AppCompatActivity; import androidx.fragment.app.FragmentManager; import com.jakewharton.rxbinding2.widget.AdapterViewItemClickEvent; @@ -35,6 +34,7 @@ import org.commcare.android.javarosa.IntentCallout; import org.commcare.android.logging.ReportingUtils; import org.commcare.cases.entity.Entity; +import org.commcare.cases.entity.EntityLoadingProgressListener; import org.commcare.cases.entity.NodeEntityFactory; import org.commcare.dalvik.R; import org.commcare.fragments.ContainerFragment; @@ -58,6 +58,7 @@ import org.commcare.utils.EntityDetailUtils; import org.commcare.utils.EntitySelectRefreshTimer; import org.commcare.utils.SerializationUtil; +import org.commcare.utils.StringUtils; import org.commcare.views.EntityView; import org.commcare.views.TabbedDetailView; import org.commcare.views.UserfacingErrorHandling; @@ -166,6 +167,8 @@ public class EntitySelectActivity extends SaveSessionCommCareActivity // Handler for displaying alert dialog when no location providers are found private final LocationNotificationHandler locationNotificationHandler = new LocationNotificationHandler(this); + private AdapterView visibleView; + private TextView progressTv; @Override public void onCreateSessionSafe(Bundle savedInstanceState) { @@ -254,7 +257,6 @@ private void setupUI(boolean isOrientationChange) { setContentView(R.layout.entity_select_layout); } - AdapterView visibleView; GridView gridView = this.findViewById(R.id.screen_entity_select_grid); ListView listView = this.findViewById(R.id.screen_entity_select_list); if (shortSelect.shouldBeLaidOutInGrid()) { @@ -268,6 +270,7 @@ private void setupUI(boolean isOrientationChange) { gridView.setVisibility(View.GONE); EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView()); } + progressTv = findViewById(R.id.progress_text); RxAdapterView.itemClickEvents(visibleView) .subscribeOn(AndroidSchedulers.mainThread()) .throttleFirst(CLICK_DEBOUNCE_TIME, TimeUnit.MILLISECONDS) @@ -356,7 +359,7 @@ private void setupUIFromAdapter(AdapterView view) { if (view instanceof ListView) { EntitySelectViewSetup.setupDivider(this, (ListView)view, shortSelect.usesEntityTileView()); } - findViewById(R.id.entity_select_loading).setVisibility(View.GONE); + findViewById(R.id.progress_container).setVisibility(View.GONE); entitySelectSearchUI.setSearchBannerState(); } @@ -476,7 +479,8 @@ public boolean loadEntities() { } if (loader == null && !EntityLoaderTask.attachToActivity(this)) { - EntityLoaderTask entityLoader = new EntityLoaderTask(shortSelect, evalContext()); + setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_initializing)); + EntityLoaderTask entityLoader = new EntityLoaderTask(shortSelect, selectDatum, evalContext()); entityLoader.attachListener(this); entityLoader.executeParallel(selectDatum.getNodeset()); return true; @@ -852,16 +856,7 @@ public void deliverLoadResult(List> entities, List references, NodeEntityFactory factory, int focusTargetIndex) { loader = null; - - AdapterView visibleView; - if (shortSelect.shouldBeLaidOutInGrid()) { - visibleView = ((GridView)this.findViewById(R.id.screen_entity_select_grid)); - } else { - ListView listView = this.findViewById(R.id.screen_entity_select_list); - EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView()); - visibleView = listView; - } - + setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_finalizing)); adapter = new EntityListAdapter(this, shortSelect, references, entities, factory, hideActionsFromEntityList, shortSelect.getCustomActions(evalContext()), inAwesomeMode); visibleView.setAdapter(adapter); @@ -883,7 +878,7 @@ public void deliverLoadResult(List> entities, } } - findViewById(R.id.entity_select_loading).setVisibility(View.GONE); + findViewById(R.id.progress_container).setVisibility(View.GONE); if (adapter != null) { // filter by additional session data (search string, callout result data) @@ -907,6 +902,10 @@ public void deliverLoadResult(List> entities, } } + private void setProgressText(String message) { + progressTv.setText(message); + } + private void restoreAdapterStateFromSession() { entitySelectSearchUI.restoreSearchString(); @@ -933,7 +932,7 @@ private void updateSelectedItem(TreeReference selected, boolean forceMove) { @Override public void attachLoader(EntityLoaderTask task) { - findViewById(R.id.entity_select_loading).setVisibility(View.VISIBLE); + findViewById(R.id.progress_container).setVisibility(View.VISIBLE); this.loader = task; } @@ -995,6 +994,25 @@ public void deliverLoadError(Exception e) { displayCaseListLoadException(e); } + @Override + public void deliverProgress(Integer[] values) { + EntityLoadingProgressListener.EntityLoadingProgressPhase phase = + EntityLoadingProgressListener.EntityLoadingProgressPhase.fromInt(values[0]); + // throttle to not update text too frequently + if (values[1] % 100 == 0) { + switch (phase) { + case PHASE_PROCESSING -> setProgressText( + StringUtils.getStringRobust(this, R.string.entity_list_processing, + new String[]{String.valueOf(values[1]), String.valueOf(values[2])})); + case PHASE_CACHING -> setProgressText( + StringUtils.getStringRobust(this, R.string.entity_list_loading_cache)); + case PHASE_UNCACHED_CALCULATION -> setProgressText( + StringUtils.getStringRobust(this, R.string.entity_list_calculating, + new String[]{String.valueOf(values[1]), String.valueOf(values[2])})); + } + } + } + @Override protected boolean onForwardSwipe() { // If user has picked an entity, move along to form entry diff --git a/app/src/org/commcare/entity/AndroidAsyncNodeEntityFactory.kt b/app/src/org/commcare/entity/AndroidAsyncNodeEntityFactory.kt new file mode 100644 index 0000000000..d45afe51bb --- /dev/null +++ b/app/src/org/commcare/entity/AndroidAsyncNodeEntityFactory.kt @@ -0,0 +1,116 @@ +package org.commcare.entity + +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.commcare.CommCareApplication +import org.commcare.cases.entity.AsyncNodeEntityFactory +import org.commcare.cases.entity.Entity +import org.commcare.cases.entity.EntityStorageCache +import org.commcare.suite.model.Detail +import org.commcare.suite.model.EntityDatum +import org.commcare.tasks.PrimeEntityCacheHelper +import org.commcare.util.LogTypes +import org.javarosa.core.model.condition.EvaluationContext +import org.javarosa.core.model.instance.TreeReference +import org.javarosa.core.services.Logger +import java.lang.RuntimeException + +/** + * Android Specific Implementation of AsyncNodeEntityFactory + * Uses [PrimeEntityCacheHelper] to prime entity cache blocking the user when required + */ +class AndroidAsyncNodeEntityFactory( + d: Detail, + private val entityDatum: EntityDatum?, + ec: EvaluationContext?, + entityStorageCache: EntityStorageCache? +) : AsyncNodeEntityFactory(d, ec, entityStorageCache) { + + companion object { + const val TEN_MINUTES = 10 * 60 * 1000L + } + + override fun prepareEntitiesInternal( + entities: MutableList> + ) { + if (detail.shouldOptimize() && detail.isCacheEnabled) { + // we only want to block if lazy load is not enabled + if (!detail.isLazyLoading) { + if (entityDatum == null) { + throw RuntimeException("Entity Datum must be defined for an async entity factory"); + } + val primeEntityCacheHelper = PrimeEntityCacheHelper.getInstance() + if (primeEntityCacheHelper.isInProgress()) { + // if we are priming something else at the moment, expedite the current detail + if (!primeEntityCacheHelper.isDatumInProgress(detail.id)) { + primeEntityCacheHelper.expediteDetailWithId( + getCurrentCommandId(), + detail, + entityDatum, + entities, + progressListener + ) + } else { + primeEntityCacheHelper.setListener(progressListener) + observePrimeCacheWork(primeEntityCacheHelper, entities) + } + } else { + // we either have not started priming or already completed. In both cases + // we want to re-prime to make sure we calculate any uncalculated data first + primeEntityCacheHelper.primeEntityCacheForDetail( + getCurrentCommandId(), + detail, + entityDatum, + entities, + progressListener + ) + } + } + } else { + super.prepareEntitiesInternal(entities) + } + } + + private fun observePrimeCacheWork( + primeEntityCacheHelper: PrimeEntityCacheHelper, + entities: MutableList> + ) { + var resultRegistered = false + while (primeEntityCacheHelper.isInProgress() && + primeEntityCacheHelper.isDatumInProgress(detail.id) + ) { + runBlocking { + try { + withTimeout(TEN_MINUTES) { + primeEntityCacheHelper.cachedEntitiesState.collect { cachedEntities -> + resultRegistered = true + if (cachedEntities != null) { + entities.clear() + entities.addAll(cachedEntities) + return@collect + } + } + } + } catch (e: TimeoutCancellationException) { + Logger.log( + LogTypes.TYPE_MAINTENANCE, + "Timeout while waiting for the prime cache worker to finish" + ) + } + } + } + if (!resultRegistered) { + Logger.log( + LogTypes.TYPE_ERROR_ASSERTION, + "Result not conveyed from Cache Prime worker to the current thread" + ) + // re-evaluate + prepareEntitiesInternal(entities); + } + } + + private fun getCurrentCommandId(): String { + return CommCareApplication.instance().currentSession.command + } +} diff --git a/app/src/org/commcare/fragments/EntitySubnodeDetailFragment.java b/app/src/org/commcare/fragments/EntitySubnodeDetailFragment.java index a66905956a..e29c35f5c6 100755 --- a/app/src/org/commcare/fragments/EntitySubnodeDetailFragment.java +++ b/app/src/org/commcare/fragments/EntitySubnodeDetailFragment.java @@ -53,7 +53,7 @@ public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle sa if (this.adapter == null && this.loader == null && !EntityLoaderTask.attachToActivity(this)) { // Set up task to fetch entity data EntityLoaderTask theLoader = - new EntityLoaderTask(detailToDisplay, getFactoryContextForRef(referenceToDisplay)); + new EntityLoaderTask(detailToDisplay, null, getFactoryContextForRef(referenceToDisplay)); theLoader.attachListener(this); theLoader.executeParallel(detailToDisplay.getNodeset().contextualize(referenceToDisplay)); @@ -79,9 +79,8 @@ public void attachLoader(EntityLoaderTask task) { } @Override - public void deliverLoadResult(List> entities, - List references, - NodeEntityFactory factory, int focusTargetIndex) { + public void deliverLoadResult(List> entities, List references, + NodeEntityFactory factory, int focusTargetIndex) { Bundle args = getArguments(); Detail detail = asw.getSession().getDetail(args.getString(DETAIL_ID)); final int thisIndex = args.getInt(CHILD_DETAIL_INDEX, -1); @@ -103,4 +102,9 @@ public void deliverLoadResult(List> entities, public void deliverLoadError(Exception e) { ((CommCareActivity)getActivity()).displayCaseListLoadException(e); } + + @Override + public void deliverProgress(Integer[] values) { + // nothing to do + } } diff --git a/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java b/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java index 2e12deadf5..cc10078f33 100755 --- a/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java +++ b/app/src/org/commcare/models/database/user/models/CommCareEntityStorageCache.java @@ -1,5 +1,8 @@ package org.commcare.models.database.user.models; +import static org.commcare.cases.entity.EntityStorageCache.ValueType.TYPE_NORMAL_FIELD; +import static org.commcare.cases.entity.EntityStorageCache.ValueType.TYPE_SORT_FIELD; + import android.content.ContentValues; import android.util.Log; @@ -18,7 +21,8 @@ import org.commcare.modern.util.Pair; import org.commcare.suite.model.Detail; import org.commcare.suite.model.DetailField; -import org.commcare.utils.SessionUnavailableException; +import org.commcare.util.LogTypes; +import org.javarosa.core.services.Logger; import java.io.Closeable; import java.util.Collection; @@ -144,11 +148,14 @@ public void invalidateCaches(Collection recordIds) { } - public int getSortFieldIdFromCacheKey(String detailId, String cacheKey) { + public int getFieldIdFromCacheKey(String detailId, String cacheKey) { + cacheKey = cacheKey.replace(String.valueOf(TYPE_SORT_FIELD), ""); + cacheKey = cacheKey.replace(String.valueOf(TYPE_NORMAL_FIELD), ""); String intId = cacheKey.substring(detailId.length() + 1); try { return Integer.parseInt(intId); } catch (NumberFormatException nfe) { + Logger.log(LogTypes.TYPE_MAINTENANCE, "Unable to parse cache key " + cacheKey); //TODO: Kill this cache key if this didn't work return -1; } @@ -185,8 +192,52 @@ public static int getEntityCacheWipedPref(String uuid) { public void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, Detail detail) { + if (detail.isCacheEnabled()) { + Vector cacheKeys = new Vector<>(); + String validKeys = buildValidKeys(cacheKeys, detail); + if (validKeys.isEmpty()) { + return; + } + + //Create our full args tree. We need the elements from the cache primer + //along with the specific keys we wanna pull out + String[] args = new String[cachePrimeKeys[1].length + 2 * cacheKeys.size()]; + System.arraycopy(cachePrimeKeys[1], 0, args, 0, cachePrimeKeys[1].length); + + for (int i = 0; i < cacheKeys.size(); ++i) { + args[cachePrimeKeys[1].length + i] = cacheKeys.get(i); + } + + String[] names = cachePrimeKeys[0]; + String whereClause = buildKeyNameWhereClause(names); + long now = System.currentTimeMillis(); + String sqlStatement = + "SELECT entity_key, cache_key, value FROM entity_cache JOIN AndroidCase ON entity_cache" + + ".entity_key = AndroidCase.commcare_sql_id WHERE " + + + whereClause + " AND " + CommCareEntityStorageCache.COL_APP_ID + " = '" + + AppUtils.getCurrentAppId() + + "' AND cache_key IN " + validKeys; + SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle(); + if (SqlStorage.STORAGE_OUTPUT_DEBUG) { + DbUtil.explainSql(db, sqlStatement, args); + } + + populateEntitySet(db, sqlStatement, args, entitySet); + + if (SqlStorage.STORAGE_OUTPUT_DEBUG) { + Log.d(TAG, "Sequential Cache Load: " + (System.currentTimeMillis() - now) + "ms"); + } + } else { + primeCacheOld(entitySet, cachePrimeKeys, detail); + } + } + + @Deprecated + private void primeCacheOld(Hashtable entitySet, String[][] cachePrimeKeys, + Detail detail) { Vector sortKeys = new Vector<>(); - String validKeys = buildValidKeys(sortKeys, detail.getFields()); + String validKeys = buildValidSortKeys(sortKeys, detail.getFields()); if ("".equals(validKeys)) { return; } @@ -198,15 +249,19 @@ public void primeCache(Hashtable entitySet, String[][] cach System.arraycopy(cachePrimeKeys[1], 0, args, 0, cachePrimeKeys[1].length); for (int i = 0; i < sortKeys.size(); ++i) { - args[cachePrimeKeys[1].length + i] = getCacheKey(detail.getId(), String.valueOf(sortKeys.get(i))); + args[cachePrimeKeys[1].length + i] = detail.getId() + "_" + sortKeys.get(i); } String[] names = cachePrimeKeys[0]; String whereClause = buildKeyNameWhereClause(names); long now = System.currentTimeMillis(); - String sqlStatement = "SELECT entity_key, cache_key, value FROM entity_cache JOIN AndroidCase ON entity_cache.entity_key = AndroidCase.commcare_sql_id WHERE " + - whereClause + " AND " + CommCareEntityStorageCache.COL_APP_ID + " = '" + AppUtils.getCurrentAppId() + - "' AND cache_key IN " + validKeys; + String sqlStatement = + "SELECT entity_key, cache_key, value FROM entity_cache JOIN AndroidCase ON entity_cache" + + ".entity_key = AndroidCase.commcare_sql_id WHERE " + + + whereClause + " AND " + CommCareEntityStorageCache.COL_APP_ID + " = '" + + AppUtils.getCurrentAppId() + + "' AND cache_key IN " + validKeys; SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle(); if (SqlStorage.STORAGE_OUTPUT_DEBUG) { DbUtil.explainSql(db, sqlStatement, args); @@ -219,11 +274,12 @@ public void primeCache(Hashtable entitySet, String[][] cach } } - public String getCacheKey(String detailId, String mFieldId) { - return detailId + "_" + mFieldId; + public String getCacheKey(String detailId, String mFieldId, ValueType valueType) { + return valueType + "_" + detailId + "_" + mFieldId; } - private static String buildValidKeys(Vector sortKeys, DetailField[] fields) { + @Deprecated + private static String buildValidSortKeys(Vector sortKeys, DetailField[] fields) { String validKeys = "("; boolean added = false; for (int i = 0; i < fields.length; ++i) { @@ -241,6 +297,27 @@ private static String buildValidKeys(Vector sortKeys, DetailField[] fie } } + private String buildValidKeys(Vector keys, Detail detail) { + StringBuilder validKeys = new StringBuilder("("); + DetailField[] fields = detail.getFields(); + for (int i = 0; i < fields.length; ++i) { + if (fields[i].isOptimize()) { + keys.add(getCacheKey(detail.getId(), String.valueOf(i), + TYPE_NORMAL_FIELD)); + validKeys.append("?, "); + if (fields[i].getSort() != null) { + keys.add(getCacheKey(detail.getId(), String.valueOf(i), + ValueType.TYPE_SORT_FIELD)); + validKeys.append("?, "); + } + } + } + if (!keys.isEmpty()) { + return validKeys.substring(0, validKeys.length() - 2) + ")"; + } + return ""; + } + private static String buildKeyNameWhereClause(String[] names) { String whereClause = ""; for (int i = 0; i < names.length; ++i) { @@ -256,11 +333,15 @@ private static void populateEntitySet(SQLiteDatabase db, String sqlStatement, St Hashtable entitySet) { Cursor walker = db.rawQuery(sqlStatement, args); while (walker.moveToNext()) { - String entityId = walker.getString(walker.getColumnIndex("entity_key")); - String cacheId = walker.getString(walker.getColumnIndex("cache_key")); - String val = walker.getString(walker.getColumnIndex("value")); - if (entitySet.containsKey(entityId)) { - entitySet.get(entityId).setSortData(cacheId, val); + String entityKey = walker.getString(walker.getColumnIndex("entity_key")); + if (entitySet.containsKey(entityKey)) { + String cacheKey = walker.getString(walker.getColumnIndex("cache_key")); + String value = walker.getString(walker.getColumnIndex("value")); + if (cacheKey.startsWith(TYPE_NORMAL_FIELD.toString())) { + entitySet.get(entityKey).setFieldData(cacheKey, value); + } else { + entitySet.get(entityKey).setSortData(cacheKey, value); + } } } walker.close(); diff --git a/app/src/org/commcare/tasks/DataPullTask.java b/app/src/org/commcare/tasks/DataPullTask.java index 7f5242b644..761ca366ec 100644 --- a/app/src/org/commcare/tasks/DataPullTask.java +++ b/app/src/org/commcare/tasks/DataPullTask.java @@ -437,6 +437,7 @@ private ResultAndError handleBadLocalState(AndroidTransactionPar if (returnCode == PROGRESS_DONE) { // Recovery was successful onSuccessfulSync(); + PrimeEntityCacheHelper.schedulePrimeEntityCacheWorker(); return new ResultAndError<>(PullTaskResult.DOWNLOAD_SUCCESS); } else if (returnCode == PROGRESS_RECOVERY_FAIL_SAFE || returnCode == PROGRESS_RECOVERY_FAIL_BAD) { wipeLoginIfItOccurred(); diff --git a/app/src/org/commcare/tasks/EntityLoaderHelper.kt b/app/src/org/commcare/tasks/EntityLoaderHelper.kt index d4b40aeaf7..361e338caa 100644 --- a/app/src/org/commcare/tasks/EntityLoaderHelper.kt +++ b/app/src/org/commcare/tasks/EntityLoaderHelper.kt @@ -5,17 +5,24 @@ import io.reactivex.functions.Cancellable import org.commcare.activities.EntitySelectActivity import org.commcare.cases.entity.AsyncNodeEntityFactory import org.commcare.cases.entity.Entity +import org.commcare.cases.entity.EntityLoadingProgressListener import org.commcare.cases.entity.EntityStorageCache import org.commcare.cases.entity.NodeEntityFactory +import org.commcare.entity.AndroidAsyncNodeEntityFactory import org.commcare.models.database.user.models.CommCareEntityStorageCache import org.commcare.preferences.DeveloperPreferences import org.commcare.suite.model.Detail +import org.commcare.suite.model.EntityDatum import org.javarosa.core.model.condition.EvaluationContext import org.javarosa.core.model.instance.TreeReference +/** + * Helper class to load entities for an entity screen + */ class EntityLoaderHelper( detail: Detail, - evalCtx: EvaluationContext + sessionDatum: EntityDatum?, + evalCtx: EvaluationContext, ) : Cancellable { var focusTargetIndex: Int = -1 @@ -24,7 +31,11 @@ class EntityLoaderHelper( init { evalCtx.addFunctionHandler(EntitySelectActivity.getHereFunctionHandler()) - if (detail.useAsyncStrategy()) { + if (detail.shouldOptimize()) { + val entityStorageCache: EntityStorageCache = CommCareEntityStorageCache("case") + factory = AndroidAsyncNodeEntityFactory(detail, sessionDatum, evalCtx, entityStorageCache) + } else if (detail.useAsyncStrategy()) { + // legacy cache and index val entityStorageCache: EntityStorageCache = CommCareEntityStorageCache("case") factory = AsyncNodeEntityFactory(detail, evalCtx, entityStorageCache) } else { @@ -38,9 +49,13 @@ class EntityLoaderHelper( /** * Loads and prepares a list of entities derived from the given nodeset */ - fun loadEntities(nodeset: TreeReference): Pair>, List>? { + fun loadEntities( + nodeset: TreeReference, + progressListener: EntityLoadingProgressListener + ): Pair>, List>? { val references = factory.expandReferenceList(nodeset) - val entities = loadEntitiesWithReferences(references) + factory.setEntityProgressListener(progressListener) + val entities = loadEntitiesWithReferences(references, progressListener) entities?.let { factory.prepareEntities(entities) factory.printAndClearTraces("build") @@ -49,15 +64,36 @@ class EntityLoaderHelper( return null } + /** + * Primes the entity cache + */ + fun cacheEntities(nodeset: TreeReference): Pair>, List> { + val references = factory.expandReferenceList(nodeset) + val entities = loadEntitiesWithReferences(references, null) + cacheEntities(entities) + return Pair>, List>(entities, references) + } + + fun cacheEntities(entities: MutableList>?) { + factory.cacheEntities(entities) + } /** * Loads a list of entities corresponding to the given references */ - private fun loadEntitiesWithReferences(references: List): MutableList>? { + private fun loadEntitiesWithReferences( + references: List, + progressListener: EntityLoadingProgressListener? + ): MutableList>? { val entities: MutableList> = ArrayList() focusTargetIndex = -1 var indexInFullList = 0 - for (ref in references) { + for ((index, ref) in references.withIndex()) { + progressListener?.publishEntityLoadingProgress( + EntityLoadingProgressListener.EntityLoadingProgressPhase.PHASE_PROCESSING, + index, + references.size + ) if (stopLoading) { return null } @@ -75,5 +111,6 @@ class EntityLoaderHelper( override fun cancel() { stopLoading = true + factory.cancelLoading() } } diff --git a/app/src/org/commcare/tasks/EntityLoaderListener.java b/app/src/org/commcare/tasks/EntityLoaderListener.java index 6c9d47f2cd..eeccfe0d6b 100644 --- a/app/src/org/commcare/tasks/EntityLoaderListener.java +++ b/app/src/org/commcare/tasks/EntityLoaderListener.java @@ -10,7 +10,9 @@ public interface EntityLoaderListener { void attachLoader(EntityLoaderTask task); void deliverLoadResult(List> entities, List references, - NodeEntityFactory factory, int focusTargetIndex); + NodeEntityFactory factory, int focusTargetIndex); void deliverLoadError(Exception e); + + void deliverProgress(Integer... values); } diff --git a/app/src/org/commcare/tasks/EntityLoaderTask.java b/app/src/org/commcare/tasks/EntityLoaderTask.java index 2f4b6ac58d..4ef721459f 100644 --- a/app/src/org/commcare/tasks/EntityLoaderTask.java +++ b/app/src/org/commcare/tasks/EntityLoaderTask.java @@ -4,8 +4,10 @@ import org.commcare.android.logging.ForceCloseLogger; import org.commcare.cases.entity.Entity; +import org.commcare.cases.entity.EntityLoadingProgressListener; import org.commcare.logging.XPathErrorLogger; import org.commcare.suite.model.Detail; +import org.commcare.suite.model.EntityDatum; import org.commcare.tasks.templates.ManagedAsyncTask; import org.javarosa.core.model.condition.EvaluationContext; import org.javarosa.core.model.instance.TreeReference; @@ -14,11 +16,15 @@ import java.util.List; +import javax.annotation.Nullable; + /** * @author ctsims */ public class EntityLoaderTask - extends ManagedAsyncTask>, List>> { + extends + ManagedAsyncTask>, List>> implements + EntityLoadingProgressListener { private final static Object lock = new Object(); private static EntityLoaderTask pendingTask = null; @@ -27,14 +33,21 @@ public class EntityLoaderTask private final EntityLoaderHelper entityLoaderHelper; private Exception mException = null; - public EntityLoaderTask(Detail detail, EvaluationContext evalCtx) { - entityLoaderHelper = new EntityLoaderHelper(detail, evalCtx); + /** + * Creates a new instance + * + * @param detail detail we want to load + * @param entityDatum entity datum corresponding to the entity list, null for entity detail screens + * @param evalCtx evaluation context + */ + public EntityLoaderTask(Detail detail, @Nullable EntityDatum entityDatum, EvaluationContext evalCtx) { + entityLoaderHelper = new EntityLoaderHelper(detail, entityDatum, evalCtx); } @Override protected Pair>, List> doInBackground(TreeReference... nodeset) { try { - return entityLoaderHelper.loadEntities(nodeset[0]); + return entityLoaderHelper.loadEntities(nodeset[0], this); } catch (XPathException xe) { XPathErrorLogger.INSTANCE.logErrorToCurrentApp(xe); Logger.exception("Error during EntityLoaderTask: " + ForceCloseLogger.getStackTrace(xe), xe); @@ -112,4 +125,16 @@ protected void onCancelled() { super.onCancelled(); entityLoaderHelper.cancel(); } + + @Override + public void publishEntityLoadingProgress(EntityLoadingProgressPhase phase, int progress, int total) { + publishProgress(phase.getValue(), progress, total); + } + + @Override + protected void onProgressUpdate(Integer... values) { + if (listener != null) { + listener.deliverProgress(values); + } + } } diff --git a/app/src/org/commcare/tasks/PrimeEntityCache.kt b/app/src/org/commcare/tasks/PrimeEntityCache.kt new file mode 100644 index 0000000000..f19ec50fdb --- /dev/null +++ b/app/src/org/commcare/tasks/PrimeEntityCache.kt @@ -0,0 +1,28 @@ +package org.commcare.tasks + +import android.content.Context +import androidx.work.Worker +import androidx.work.WorkerParameters +import org.javarosa.core.services.Logger + +/** + * Android Worker to prime cache for entity screens + */ +class PrimeEntityCache(appContext: Context, workerParams: WorkerParameters) : Worker(appContext, workerParams) { + + override fun doWork(): Result { + try { + PrimeEntityCacheHelper.getInstance().primeEntityCache() + return Result.success() + } catch (e: Exception) { + Logger.exception("Error while priming cache in worker", e) + } finally { + PrimeEntityCacheHelper.getInstance().clearState(); + } + return Result.failure() + } + + override fun onStopped() { + PrimeEntityCacheHelper.getInstance().cancel() + } +} diff --git a/app/src/org/commcare/tasks/PrimeEntityCacheHelper.kt b/app/src/org/commcare/tasks/PrimeEntityCacheHelper.kt new file mode 100644 index 0000000000..117995361e --- /dev/null +++ b/app/src/org/commcare/tasks/PrimeEntityCacheHelper.kt @@ -0,0 +1,215 @@ +package org.commcare.tasks + +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequest +import androidx.work.WorkManager +import io.reactivex.functions.Cancellable +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import org.commcare.CommCareApplication +import org.commcare.cases.entity.Entity +import org.commcare.cases.entity.EntityLoadingProgressListener +import org.commcare.suite.model.Detail +import org.commcare.suite.model.EntityDatum +import org.commcare.utils.AndroidCommCarePlatform +import org.javarosa.core.model.condition.EvaluationContext +import org.javarosa.core.model.instance.TreeReference +import org.javarosa.core.services.Logger +import org.javarosa.xpath.XPathException + +/** + * Helper to prime cache for all entity screens in the app + * + * Implemented as a singleton to restrict caller from starting another + * cache prime process if one is already in progress. Therefore it's advisable + * to initiate all caching operations using this class + */ +class PrimeEntityCacheHelper private constructor() : Cancellable { + + private var entityLoaderHelper: EntityLoaderHelper? = null + + @Volatile + private var inProgress = false + + @Volatile + private var currentDatumInProgress: String? = null + private var listener: EntityLoadingProgressListener? = null + + private val _cachedEntitiesState = MutableStateFlow>?>(null) + val cachedEntitiesState: StateFlow>?> get() = _cachedEntitiesState + + + companion object { + @Volatile + private var instance: PrimeEntityCacheHelper? = null + + private const val PRIME_ENTITY_CACHE_REQUEST = "prime-entity-cache-request" + + @JvmStatic + fun getInstance() = + instance ?: synchronized(PrimeEntityCacheHelper::class) { + instance ?: PrimeEntityCacheHelper().also { instance = it } + } + + /** + * Schedules a background worker request to prime cache for all + * cache backed entity list screens in the current seated app + */ + @JvmStatic + fun schedulePrimeEntityCacheWorker() { + val primeEntityCacheRequest = OneTimeWorkRequest.Builder(PrimeEntityCache::class.java).build() + WorkManager.getInstance(CommCareApplication.instance()) + .enqueueUniqueWork( + PRIME_ENTITY_CACHE_REQUEST, + ExistingWorkPolicy.KEEP, + primeEntityCacheRequest + ) + } + + @JvmStatic + fun cancelWork() { + instance?.cancel() + WorkManager.getInstance(CommCareApplication.instance()).cancelUniqueWork(PRIME_ENTITY_CACHE_REQUEST) + } + } + + /** + * Primes cache for all entity screens in the app + * @throws IllegalStateException if a cache prime is already in progress or user session is not active + */ + @Synchronized + fun primeEntityCache() { + checkPreConditions() + try { + primeEntityCacheForApp(CommCareApplication.instance().commCarePlatform) + } finally { + clearState() + } + } + + /** + * Primes cache for given entities set against the [detail] + * @throws IllegalStateException if a cache prime is already in progress or user session is not active + */ + @Synchronized + fun primeEntityCacheForDetail( + commandId: String, + detail: Detail, + entityDatum: EntityDatum, + entities: MutableList>, + progressListener: EntityLoadingProgressListener + ) { + checkPreConditions() + try { + primeCacheForDetail(commandId, detail, entityDatum, entities, progressListener) + } finally { + clearState() + } + } + + /** + * Cancel any current cache prime process to expedite cache calculations for given [detail] + * Reschedules the work again in background afterwards + */ + @Synchronized + fun expediteDetailWithId( + commandId: String, + detail: Detail, + entityDatum: EntityDatum, + entities: MutableList>, + progressListener: EntityLoadingProgressListener + ) { + cancel() + primeEntityCacheForDetail(commandId, detail, entityDatum, entities, progressListener) + schedulePrimeEntityCacheWorker() + } + + fun isDatumInProgress(datumId: String): Boolean { + return currentDatumInProgress?.contentEquals(datumId) ?: false + } + + private fun primeEntityCacheForApp(commCarePlatform: AndroidCommCarePlatform) { + inProgress = true + val commandMap = commCarePlatform.commandToEntryMap + for (command in commandMap.keys()) { + val entry = commandMap[command]!! + val sessionDatums = entry.sessionDataReqs + for (sessionDatum in sessionDatums) { + if (sessionDatum is EntityDatum) { + val shortDetailId = sessionDatum.shortDetail + if (shortDetailId != null) { + val detail = commCarePlatform.getDetail(shortDetailId) + try { + primeCacheForDetail(entry.commandId, detail, sessionDatum) + } catch (e: XPathException) { + // Bury any xpath exceptions here as we don't want to hold off priming cache + // for other datums because of an error with a particular detail. + Logger.exception( + "Xpath error on trying to prime cache for datum: " + sessionDatum.dataId, + e + ) + } + } + } + } + } + } + + private fun primeCacheForDetail( + commandId: String, + detail: Detail, + entityDatum: EntityDatum, + entities: MutableList>? = null, + progressListener: EntityLoadingProgressListener? = null + ) { + if (!detail.isCacheEnabled()) return + currentDatumInProgress = entityDatum.dataId + entityLoaderHelper = EntityLoaderHelper(detail, entityDatum, evalCtx(commandId)).also { + it.factory.setEntityProgressListener(progressListener) + } + // Handle the cache operation based on the available input + val cachedEntities = when { + entities != null -> { + entityLoaderHelper!!.cacheEntities(entities) + entities + } + + else -> entityLoaderHelper!!.cacheEntities(entityDatum.nodeset).first + } + _cachedEntitiesState.value = cachedEntities + currentDatumInProgress = null + } + + private fun evalCtx(commandId: String): EvaluationContext { + return CommCareApplication.instance().currentSessionWrapper.getRestrictedEvaluationContext(commandId, null) + } + + /** + * Clears any volatile state and nullify the singleton instance + */ + fun clearState() { + entityLoaderHelper = null + inProgress = false + listener = null + currentDatumInProgress = null + instance = null + } + + private fun checkPreConditions() { + require(CommCareApplication.instance().session.isActive) { "User session must be active to prime entity cache" } + require(!inProgress) { "We are already priming the cache" } + } + + override fun cancel() { + entityLoaderHelper?.cancel() + clearState() + } + + fun isInProgress(): Boolean { + return inProgress + } + + fun setListener(entityLoadingProgressListener: EntityLoadingProgressListener) { + entityLoaderHelper?.factory?.setEntityProgressListener(entityLoadingProgressListener) + } +} diff --git a/app/unit-tests/src/org/commcare/CommCareTestApplication.java b/app/unit-tests/src/org/commcare/CommCareTestApplication.java index eb38ef32cb..67ffd03705 100644 --- a/app/unit-tests/src/org/commcare/CommCareTestApplication.java +++ b/app/unit-tests/src/org/commcare/CommCareTestApplication.java @@ -47,6 +47,9 @@ import androidx.annotation.NonNull; import androidx.test.core.app.ApplicationProvider; +import androidx.work.Configuration; +import androidx.work.WorkManager; + import okhttp3.MultipartBody; import okhttp3.RequestBody; @@ -95,6 +98,15 @@ protected void turnOnStrictMode() { .build()); } + + public static void initWorkManager() { + Context context = ApplicationProvider.getApplicationContext(); + Configuration config = new Configuration.Builder() + .setMinimumLoggingLevel(Log.DEBUG) + .build(); + WorkManager.initialize(context, config); + } + @Override public HybridFileBackedSqlStorage getFileBackedAppStorage(String name, Class c) { return getCurrentApp().getFileBackedStorage(name, c); diff --git a/app/unit-tests/src/org/commcare/android/tests/DataPullTaskTest.java b/app/unit-tests/src/org/commcare/android/tests/DataPullTaskTest.java index 93defd2edf..2a9b169d37 100644 --- a/app/unit-tests/src/org/commcare/android/tests/DataPullTaskTest.java +++ b/app/unit-tests/src/org/commcare/android/tests/DataPullTaskTest.java @@ -1,5 +1,10 @@ package org.commcare.android.tests; +import static org.commcare.CommCareTestApplication.initWorkManager; + +import android.content.Context; +import android.util.Log; + import org.commcare.CommCareApp; import org.commcare.CommCareApplication; import org.commcare.CommCareTestApplication; @@ -17,6 +22,8 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.work.Configuration; +import androidx.work.WorkManager; /** * Coverage for different DataPullTask codepaths. @@ -91,6 +98,7 @@ public void dataPullRecoverFailTest() { @Test public void dataPullRecoverWithRetryTest() { + initWorkManager(); installLoginAndUseLocalKeys(); runDataPull(new Integer[]{412, 202, 200}, new String[]{GOOD_RESTORE, RETRY_RESPONSE, GOOD_RESTORE}); Assert.assertEquals(DataPullTask.PullTaskResult.DOWNLOAD_SUCCESS, dataPullResult.data);