Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache and Lazy-Load Modifications #2928

Draft
wants to merge 19 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2518b23
Adds a worker task to prime entity cache
shubham1g5 Dec 9, 2024
878c09c
Schedule prime cache task after 412 and on initializing user session
shubham1g5 Dec 9, 2024
c1e6a35
Set up progress indicator on entity list screen
shubham1g5 Dec 11, 2024
9aa1220
Creates a Android specific async factory to bind to any existing prim…
shubham1g5 Dec 19, 2024
c4a68a6
Move methods to schedule/cancel cache work to cache helper
shubham1g5 Dec 19, 2024
d78960d
Re-schedule cancelled work when expediting a detail sepcific cache
shubham1g5 Dec 19, 2024
dd05545
Improve progress tracking by providing information on individual enti…
shubham1g5 Dec 23, 2024
acd3aa6
Handle for new cache config while initiating entity factory
shubham1g5 Dec 24, 2024
7122c92
Cancellation handling
shubham1g5 Dec 24, 2024
63743aa
Always clear state after prime cache worker run
shubham1g5 Dec 24, 2024
ede6caf
correct eval context by including command id
shubham1g5 Dec 24, 2024
f145dd6
Fixes progress tracking plus tracking the live worker in async task
shubham1g5 Dec 25, 2024
8b53cbf
Uses datum id to track progress instead of detail as a detail can pot…
shubham1g5 Dec 26, 2024
1e8dfd3
Thread safety
shubham1g5 Dec 26, 2024
32d554e
lint
shubham1g5 Dec 26, 2024
90b0c96
Make clear distinction in between old and new cache and index switch
shubham1g5 Jan 6, 2025
ab5d325
Priming cache for normal and sort values for the detail fields marked…
shubham1g5 Jan 9, 2025
6c9be03
Tweaks
shubham1g5 Jan 10, 2025
dbbcfd4
fixes test dependent on work manager init
shubham1g5 Jan 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions app/res/layout/entity_select_layout.xml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
android:visibility="gone"/>

<RelativeLayout
android:id="@+id/progress_container"
android:layout_width="match_parent"
android:layout_height="fill_parent"
android:layout_below="@id/entity_select_filter_dropdown">
Expand All @@ -63,8 +64,18 @@
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_centerInParent="true"/>

<TextView
android:id="@+id/progress_text"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_below="@+id/entity_select_loading"
android:padding="8dp"
android:layout_centerInParent="true" />

</RelativeLayout>


<FrameLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
Expand Down
3 changes: 3 additions & 0 deletions app/res/values/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -456,4 +456,7 @@
<string name="fcm_default_notification_channel">notification-channel-push-notifications</string>
<string name="app_with_id_not_found">Required CommCare App is not installed on device</string>
<string name="audio_recording_notification">Audio Recording Notification</string>
<string name="entity_list_initializing" cc:translatable="true">Initializing list…</string>
<string name="entity_list_processing" cc:translatable="true">Processing %1s out of %2s</string>
<string name="entity_list_finalizing" cc:translatable="true">Finalizing list…</string>
</resources>
5 changes: 4 additions & 1 deletion app/src/org/commcare/CommCareApplication.java
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@
import org.commcare.tasks.DataPullTask;
import org.commcare.tasks.DeleteLogs;
import org.commcare.tasks.LogSubmissionTask;
import org.commcare.tasks.PrimeEntityCache;
import org.commcare.tasks.PrimeEntityCacheHelper;
import org.commcare.tasks.PurgeStaleArchivedFormsTask;
import org.commcare.tasks.templates.ManagedAsyncTask;
import org.commcare.update.UpdateHelper;
Expand Down Expand Up @@ -387,6 +389,7 @@ protected void cancelWorkManagerTasks() {
if (currentApp != null) {
WorkManager.getInstance(this).cancelUniqueWork(
FormSubmissionHelper.getFormSubmissionRequestName(currentApp.getUniqueId()));
PrimeEntityCacheHelper.cancelWork();
}
}

Expand Down Expand Up @@ -796,7 +799,7 @@ public void onServiceConnected(ComponentName className, IBinder service) {

purgeLogs();
cleanRawMedia();

PrimeEntityCacheHelper.schedulePrimeEntityCacheWorker();
}

TimedStatsTracker.registerStartSession();
Expand Down
37 changes: 22 additions & 15 deletions app/src/org/commcare/activities/EntitySelectActivity.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import android.widget.TextView;
import android.widget.Toast;

import androidx.appcompat.app.AppCompatActivity;
import androidx.fragment.app.FragmentManager;

import com.jakewharton.rxbinding2.widget.AdapterViewItemClickEvent;
Expand Down Expand Up @@ -58,6 +57,7 @@
import org.commcare.utils.EntityDetailUtils;
import org.commcare.utils.EntitySelectRefreshTimer;
import org.commcare.utils.SerializationUtil;
import org.commcare.utils.StringUtils;
import org.commcare.views.EntityView;
import org.commcare.views.TabbedDetailView;
import org.commcare.views.UserfacingErrorHandling;
Expand Down Expand Up @@ -166,6 +166,8 @@ public class EntitySelectActivity extends SaveSessionCommCareActivity
// Handler for displaying alert dialog when no location providers are found
private final LocationNotificationHandler locationNotificationHandler =
new LocationNotificationHandler(this);
private AdapterView visibleView;
private TextView progressTv;

@Override
public void onCreateSessionSafe(Bundle savedInstanceState) {
Expand Down Expand Up @@ -254,7 +256,6 @@ private void setupUI(boolean isOrientationChange) {
setContentView(R.layout.entity_select_layout);
}

AdapterView visibleView;
GridView gridView = this.findViewById(R.id.screen_entity_select_grid);
ListView listView = this.findViewById(R.id.screen_entity_select_list);
if (shortSelect.shouldBeLaidOutInGrid()) {
Expand All @@ -268,6 +269,7 @@ private void setupUI(boolean isOrientationChange) {
gridView.setVisibility(View.GONE);
EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView());
}
progressTv = findViewById(R.id.progress_text);
RxAdapterView.itemClickEvents(visibleView)
.subscribeOn(AndroidSchedulers.mainThread())
.throttleFirst(CLICK_DEBOUNCE_TIME, TimeUnit.MILLISECONDS)
Expand Down Expand Up @@ -356,7 +358,7 @@ private void setupUIFromAdapter(AdapterView view) {
if (view instanceof ListView) {
EntitySelectViewSetup.setupDivider(this, (ListView)view, shortSelect.usesEntityTileView());
}
findViewById(R.id.entity_select_loading).setVisibility(View.GONE);
findViewById(R.id.progress_container).setVisibility(View.GONE);
entitySelectSearchUI.setSearchBannerState();
}

Expand Down Expand Up @@ -476,6 +478,7 @@ public boolean loadEntities() {
}

if (loader == null && !EntityLoaderTask.attachToActivity(this)) {
setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_initializing));
EntityLoaderTask entityLoader = new EntityLoaderTask(shortSelect, evalContext());
entityLoader.attachListener(this);
entityLoader.executeParallel(selectDatum.getNodeset());
Expand Down Expand Up @@ -852,16 +855,7 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
List<TreeReference> references,
NodeEntityFactory factory, int focusTargetIndex) {
loader = null;

AdapterView visibleView;
if (shortSelect.shouldBeLaidOutInGrid()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this removing the gridview modality of entity selects entirely? I think it is mostly deprecated, but probably important to call out specifically if so to confirm that it's safe to do so.

visibleView = ((GridView)this.findViewById(R.id.screen_entity_select_grid));
} else {
ListView listView = this.findViewById(R.id.screen_entity_select_list);
EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView());
visibleView = listView;
}

setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_finalizing));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This message doesn't seem like the right one to me. Isn't this just the start of the loading process?

adapter = new EntityListAdapter(this, shortSelect, references, entities, factory,
hideActionsFromEntityList, shortSelect.getCustomActions(evalContext()), inAwesomeMode);
visibleView.setAdapter(adapter);
Expand All @@ -883,7 +877,7 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
}
}

findViewById(R.id.entity_select_loading).setVisibility(View.GONE);
findViewById(R.id.progress_container).setVisibility(View.GONE);

if (adapter != null) {
// filter by additional session data (search string, callout result data)
Expand All @@ -907,6 +901,10 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
}
}

private void setProgressText(String message) {
progressTv.setText(message);
}

private void restoreAdapterStateFromSession() {
entitySelectSearchUI.restoreSearchString();

Expand All @@ -933,7 +931,7 @@ private void updateSelectedItem(TreeReference selected, boolean forceMove) {

@Override
public void attachLoader(EntityLoaderTask task) {
findViewById(R.id.entity_select_loading).setVisibility(View.VISIBLE);
findViewById(R.id.progress_container).setVisibility(View.VISIBLE);
this.loader = task;
}

Expand Down Expand Up @@ -995,6 +993,15 @@ public void deliverLoadError(Exception e) {
displayCaseListLoadException(e);
}

@Override
public void deliverProgress(Integer[] values) {
// throttle to not update text too frequently
if (values[0] % 100 == 0) {
setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_processing,
new String[]{String.valueOf(values[0]), String.valueOf(values[1])}));
}
}

@Override
protected boolean onForwardSwipe() {
// If user has picked an entity, move along to form entry
Expand Down
90 changes: 90 additions & 0 deletions app/src/org/commcare/entity/AndroidAsyncNodeEntityFactory.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package org.commcare.entity

import android.util.Pair
import org.commcare.cases.entity.AsyncNodeEntityFactory
import org.commcare.cases.entity.Entity
import org.commcare.cases.entity.EntityStorageCache
import org.commcare.suite.model.Detail
import org.commcare.tasks.PrimeEntityCacheHelper
import org.commcare.util.LogTypes
import org.javarosa.core.model.condition.EvaluationContext
import org.javarosa.core.model.instance.TreeReference
import org.javarosa.core.services.Logger

/**
* Android Specific Implementation of AsyncNodeEntityFactory
* Uses [PrimeEntityCacheHelper] to prime entity cache blocking the user when required
*/
class AndroidAsyncNodeEntityFactory(d: Detail, ec: EvaluationContext?, entityStorageCache: EntityStorageCache?) :
AsyncNodeEntityFactory(d, ec, entityStorageCache), PrimeEntityCacheListener {

companion object {
const val TWO_MINUTES = 2 * 60 * 1000
}

private var cachedEntities: List<Entity<TreeReference>>? = null
private var completedCachePrime = false

override fun prepareEntitiesInternal(entities: MutableList<Entity<TreeReference>>) {
if (detail.shouldCache()) {
// we only want to block if lazy load is not enabled
if (!detail.shouldLazyLoad()) {
val primeEntityCacheHelper = PrimeEntityCacheHelper.getInstance()
if (primeEntityCacheHelper.isInProgress()) {
// if we are priming something else at the moment, expedite the current detail
if (!primeEntityCacheHelper.isDetailInProgress(detail.id)) {
primeEntityCacheHelper.expediteDetailWithId(detail, entities)
} else {
// otherwise wait for existing prime process to complete
primeEntityCacheHelper.setListener(this)
waitForCachePrimeWork(entities, primeEntityCacheHelper)
if (cachedEntities != null) {
entities.clear()
entities.addAll(cachedEntities!!)
}
}
} else {
// we either have not started priming or already completed. In both cases
// we want to re-prime to make sure we calculate any uncalculated data first
primeEntityCacheHelper.primeEntityCacheForDetail(detail, entities)
}
}
} else {
super.prepareEntitiesInternal(entities)
}
}

private fun waitForCachePrimeWork(
entities: MutableList<Entity<TreeReference>>,
primeEntityCacheHelper: PrimeEntityCacheHelper
) {
val startTime = System.currentTimeMillis()
while (!completedCachePrime && (System.currentTimeMillis() - startTime) < TWO_MINUTES) {
// wait for it to be completed
try {
Thread.sleep(100)
} catch (_: InterruptedException) {
}
}
if (!completedCachePrime) {
Logger.log(LogTypes.TYPE_MAINTENANCE, "Still Waiting for cache priming work to complete")
// confirm if we are still priming in the worker. If yes, wait more
// otherwise recall prepareEntitiesInternal to re-evaluate the best thing to do
if (primeEntityCacheHelper.isInProgress() && primeEntityCacheHelper.isDetailInProgress(detail.id)) {
waitForCachePrimeWork(entities, primeEntityCacheHelper)
} else {
prepareEntitiesInternal(entities)
}
}
}
shubham1g5 marked this conversation as resolved.
Show resolved Hide resolved

override fun onPrimeEntityCacheComplete(
currentDetailInProgress: String,
cachedEntitiesWithRefs: Pair<List<Entity<TreeReference>>, List<TreeReference>>
) {
if (detail.id!!.contentEquals(currentDetailInProgress)) {
cachedEntities = cachedEntitiesWithRefs.first
completedCachePrime = true
}
}
}
13 changes: 13 additions & 0 deletions app/src/org/commcare/entity/PrimeEntityCacheListener.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.commcare.entity

import android.util.Pair
import org.commcare.cases.entity.Entity
import org.javarosa.core.model.instance.TreeReference

interface PrimeEntityCacheListener {

fun onPrimeEntityCacheComplete(
currentDetailInProgress: String,
cachedEntitiesWithRefs: Pair<List<Entity<TreeReference>>, List<TreeReference>>
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,9 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
public void deliverLoadError(Exception e) {
((CommCareActivity)getActivity()).displayCaseListLoadException(e);
}

@Override
public void deliverProgress(Integer[] values) {
// nothing to do
}
}
1 change: 1 addition & 0 deletions app/src/org/commcare/tasks/DataPullTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,7 @@ private ResultAndError<PullTaskResult> handleBadLocalState(AndroidTransactionPar
if (returnCode == PROGRESS_DONE) {
// Recovery was successful
onSuccessfulSync();
PrimeEntityCacheHelper.schedulePrimeEntityCacheWorker();
return new ResultAndError<>(PullTaskResult.DOWNLOAD_SUCCESS);
} else if (returnCode == PROGRESS_RECOVERY_FAIL_SAFE || returnCode == PROGRESS_RECOVERY_FAIL_BAD) {
wipeLoginIfItOccurred();
Expand Down
31 changes: 26 additions & 5 deletions app/src/org/commcare/tasks/EntityLoaderHelper.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import org.commcare.cases.entity.AsyncNodeEntityFactory
import org.commcare.cases.entity.Entity
import org.commcare.cases.entity.EntityStorageCache
import org.commcare.cases.entity.NodeEntityFactory
import org.commcare.entity.AndroidAsyncNodeEntityFactory
import org.commcare.models.database.user.models.CommCareEntityStorageCache
import org.commcare.preferences.DeveloperPreferences
import org.commcare.suite.model.Detail
Expand All @@ -26,7 +27,7 @@ class EntityLoaderHelper(
evalCtx.addFunctionHandler(EntitySelectActivity.getHereFunctionHandler())
if (detail.useAsyncStrategy()) {
val entityStorageCache: EntityStorageCache = CommCareEntityStorageCache("case")
factory = AsyncNodeEntityFactory(detail, evalCtx, entityStorageCache)
factory = AndroidAsyncNodeEntityFactory(detail, evalCtx, entityStorageCache)
} else {
factory = NodeEntityFactory(detail, evalCtx)
if (DeveloperPreferences.collectAndDisplayEntityTraces()) {
Expand All @@ -38,9 +39,12 @@ class EntityLoaderHelper(
/**
* Loads and prepares a list of entities derived from the given nodeset
*/
fun loadEntities(nodeset: TreeReference): Pair<List<Entity<TreeReference>>, List<TreeReference>>? {
fun loadEntities(
nodeset: TreeReference,
progressListener: EntityLoadingProgressListener
): Pair<List<Entity<TreeReference>>, List<TreeReference>>? {
val references = factory.expandReferenceList(nodeset)
val entities = loadEntitiesWithReferences(references)
val entities = loadEntitiesWithReferences(references, progressListener)
entities?.let {
factory.prepareEntities(entities)
factory.printAndClearTraces("build")
Expand All @@ -49,15 +53,32 @@ class EntityLoaderHelper(
return null
}

/**
* Primes the entity cache
*/
fun cacheEntities(nodeset: TreeReference): Pair<List<Entity<TreeReference>>, List<TreeReference>> {
val references = factory.expandReferenceList(nodeset)
val entities = loadEntitiesWithReferences(references, null)
cacheEntities(entities)
return Pair<List<Entity<TreeReference>>, List<TreeReference>>(entities, references)
}

fun cacheEntities(entities: MutableList<Entity<TreeReference>>?) {
factory.cacheEntities(entities)
}

/**
* Loads a list of entities corresponding to the given references
*/
private fun loadEntitiesWithReferences(references: List<TreeReference>): MutableList<Entity<TreeReference>>? {
private fun loadEntitiesWithReferences(
references: List<TreeReference>,
progressListener: EntityLoadingProgressListener?
): MutableList<Entity<TreeReference>>? {
val entities: MutableList<Entity<TreeReference>> = ArrayList()
focusTargetIndex = -1
var indexInFullList = 0
for (ref in references) {
for ((index, ref) in references.withIndex()) {
progressListener?.publishEntityLoadingProgress(index, references.size)
if (stopLoading) {
return null
}
Expand Down
2 changes: 2 additions & 0 deletions app/src/org/commcare/tasks/EntityLoaderListener.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@ void deliverLoadResult(List<Entity<TreeReference>> entities, List<TreeReference>
NodeEntityFactory factory, int focusTargetIndex);

void deliverLoadError(Exception e);

void deliverProgress(Integer... values);
}
Loading
Loading