Skip to content

Commit

Permalink
Handle Merge on EventId for EventNotes and Speed up NarrativeCache po…
Browse files Browse the repository at this point in the history
…pulation for iterative updates (#243)

* added logic for separating rows containing eventNoteIds

* added isUpdate logic to getBulkSuperPkgs

* added isUpdate logic to getBulkSuperPkgs

* Merge alt key

* error handling

* added logic for separating rows containing eventNoteIds

* added isUpdate logic to getBulkSuperPkgs

* changed order of method calls

* rebased with commits for alternate key

* removed space

* removed getEventNote method

* EventNotesDataIterator

* Remove unused variable

---------

Co-authored-by: Marty Pradere <martyp@labkey.com>
  • Loading branch information
jallentxbiomed and labkey-martyp authored Nov 1, 2024
1 parent 9d9abe3 commit c7708e4
Show file tree
Hide file tree
Showing 7 changed files with 163 additions and 20 deletions.
4 changes: 2 additions & 2 deletions api-src/org/labkey/api/snd/SNDSequencer.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ public enum SNDSequencer
PKGID ("org.labkey.snd.api.Package", 10000),
SUPERPKGID ("org.labkey.snd.api.SuperPackage", 10000),
CATEGORYID ("org.labkey.snd.api.Categories", 100),
PROJECTID ("org.labkey.snd.api.Project", 1000),
PROJECTID ("org.labkey.snd.api.Project", 10000),
PROJECTITEMID ("org.labkey.snd.api.ProjectItem", 30000),
EVENTID ("org.labkey.snd.api.Event", 2000000),
EVENTDATAID ("org.labkey.snd.api.EventData", 3500000);
EVENTDATAID ("org.labkey.snd.api.EventData", 5000000);

private String sequenceName;
private int minId;
Expand Down
2 changes: 1 addition & 1 deletion api-src/org/labkey/api/snd/SNDService.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ static SNDService get()
void fillInNarrativeCache(Container c, User u, Logger logger);
void clearNarrativeCache(Container c, User u);
void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Object>> eventIds);
void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger);
void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger, boolean isFullReload);
Map<Integer, Category> getAllCategories(Container c, User u);
Integer getQCStateId(Container c, User u, QCStateEnum qcState);
QCStateEnum getQCState(Container c, User u, int qcStateId);
Expand Down
32 changes: 19 additions & 13 deletions src/org/labkey/snd/SNDManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -3113,10 +3113,10 @@ public int updateNarrativeCache(Container container, User user, Set<Integer>cach
log.info("Deleting affected narrative cache rows.");
deleteNarrativeCacheRows(container, user, rows, errors);
//repopulate
if (isUpdate)
if (!errors.hasErrors() && isUpdate)
{
log.info("Repopulate affected rows in narrative cache.");
populateNarrativeCache(container, user, eventIds, errors, log);
populateNarrativeCache(container, user, eventIds, errors, log, false);
}
}

Expand All @@ -3142,7 +3142,7 @@ public void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Objec
}
catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e)
{
e.printStackTrace();
errors.addRowError(new ValidationException(e.getMessage()));
}
}

Expand Down Expand Up @@ -3182,13 +3182,13 @@ public void fillInNarrativeCache(Container c, User u, BatchValidationException e

List<Integer> eventIds = selector.getArrayList(Integer.class);

populateNarrativeCache(c, u, eventIds, errors, logger);
populateNarrativeCache(c, u, eventIds, errors, logger, true);
}

/**
* Populate specific event narratives in narrative cache.
*/
public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, BatchValidationException errors, @Nullable Logger logger)
public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, BatchValidationException errors, @Nullable Logger logger, boolean isFullReload)
{
UserSchema sndSchema = getSndUserSchemaAdminRole(c, u);
QueryUpdateService eventsCacheQus = getNewQueryUpdateService(sndSchema, SNDSchema.EVENTSCACHE_TABLE_NAME);
Expand All @@ -3208,7 +3208,7 @@ public void populateNarrativeCache(Container c, User u, List<Integer> eventIds,
logger.info("Generating narratives.");
}

Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, null, lookups, errors);
Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, isFullReload ? null : eventIds, lookups, errors, isFullReload);

AtomicInteger count = new AtomicInteger(0);

Expand Down Expand Up @@ -3801,7 +3801,7 @@ public Event getEvent(Container c, User u, int eventId, Set<EventNarrativeOption
List<GWTPropertyDescriptor> packageExtraFields = getExtraFields(c, u, SNDSchema.PKGS_TABLE_NAME);

// Retrieve all SuperPackages associated with a single event
Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, eventId, lookups, errors);
Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, Collections.singletonList(eventId), lookups, errors, false);

// Map top-level EventDataIds to associated SuperPackage objects and group by EventId
Map<Integer, Map<Integer, SuperPackage>> topLevelEventDataSuperPkgs = getBulkTopLevelEventDataSuperPkgs(c, u, Collections.singletonList(eventId), superPackages);
Expand Down Expand Up @@ -3911,14 +3911,14 @@ public Map<Integer, Event> getBulkEvents(Container c, User u, List<Integer> even
* @param u User object representing the current user.
* @param packageExtraFields Extra fields related to the SuperPackage that need to be included in the response.
* @param pkgQus QueryUpdateService for handling SuperPackage updates.
* @param eventId The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages.
* @param eventIds The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages.
* @param lookups Map for additional lookup criteria or filtering, used to retrieve SuperPackages.
* @param errors BatchValidationException object used to accumulate any errors encountered during the process.
*
* @return A map of SuperPackage objects, keyed by SuperPkgId.
*/
private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GWTPropertyDescriptor> packageExtraFields, QueryUpdateService pkgQus, @Nullable Integer eventId,
Map<String, String> lookups, BatchValidationException errors) {
private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GWTPropertyDescriptor> packageExtraFields, QueryUpdateService pkgQus, @Nullable List<Integer> eventIds,
Map<String, String> lookups, BatchValidationException errors, boolean isFullReload) {

UserSchema schema = getSndUserSchema(c, u);

Expand All @@ -3929,10 +3929,15 @@ private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GW
sql.append(schema.getTable(SNDSchema.SUPERPKGS_TABLE_NAME), "sp");
sql.append(" JOIN " + SNDSchema.NAME + "." + SNDSchema.PKGS_TABLE_NAME + " pkg");
sql.append(" ON sp.PkgId = pkg.PkgId ");
if (eventId != null) {
if (eventIds != null) {
sql.append(" INNER JOIN " + SNDSchema.NAME + "." + SNDSchema.EVENTDATA_TABLE_NAME + " ed");
sql.append(" ON ed.SuperPkgId = sp.SuperPkgId ");
sql.append(" WHERE ed.EventId = ? ").add(eventId);
sql.append(" WHERE ed.EventId IN ( ");
ArrayDeque<Integer> eventIdsQueue = new ArrayDeque<>(eventIds);
while (eventIdsQueue.size() > 1) {
sql.append("?, ").add(eventIdsQueue.pop());
}
sql.append("?) ").add(eventIdsQueue.pop());
}
sql.append(" ORDER BY sp.SuperPkgId ");

Expand All @@ -3943,7 +3948,7 @@ private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GW

List<SuperPackage> fullTreeSuperPkgs;

if (eventId != null) {
if (!isFullReload) {
fullTreeSuperPkgs = new ArrayList<SuperPackage>();
pkgIds.forEach(pkgId -> {
SQLFragment packageSql = new SQLFragment("SELECT * FROM ");
Expand Down Expand Up @@ -4328,6 +4333,7 @@ private Map<Integer, String> getBulkEventNotes(Container c, User u, List<Integer
return eventNotesById;
}


/**
* Query the Projects table and retrieve the ID concatenated with RevisionNum for a set of objectIds
*
Expand Down
5 changes: 3 additions & 2 deletions src/org/labkey/snd/SNDServiceImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -558,11 +558,12 @@ public void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Objec
}

@Override
public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger)

public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger, boolean isFullReload)
{
BatchValidationException errors = new BatchValidationException();

SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger);
SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger, isFullReload);

if (errors.hasErrors())
throw new ApiUsageException(errors);
Expand Down
91 changes: 91 additions & 0 deletions src/org/labkey/snd/query/EventNotesDataIterator.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package org.labkey.snd.query;

import org.apache.logging.log4j.Logger;
import org.labkey.api.data.ColumnInfo;
import org.labkey.api.data.Container;
import org.labkey.api.data.DbScope;
import org.labkey.api.dataiterator.AbstractDataIterator;
import org.labkey.api.dataiterator.DataIterator;
import org.labkey.api.dataiterator.DataIteratorContext;
import org.labkey.api.dataiterator.DataIteratorUtil;
import org.labkey.api.query.BatchValidationException;
import org.labkey.api.query.UserSchema;
import org.labkey.api.security.User;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.snd.SNDManager;

import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

public class EventNotesDataIterator extends AbstractDataIterator
{
private static final SNDManager _sndManager = SNDManager.get();
private static final String EVENT_ID_COL = "eventId";
private User _user;
private Container _container;
private int _eventIdColIndex;
private Set<Integer> _eventIds = new HashSet<>();
private DataIterator _in;
private Logger log = LogHelper.getLogger(EventNotesDataIterator.class, "Fill out event notes");

public static DataIterator wrap(DataIterator in, DataIteratorContext context, Container c, User u)
{
return new EventNotesDataIterator(in, context, c, u);
}

private EventNotesDataIterator(DataIterator in, DataIteratorContext context, Container c, User u)
{
super(context);
_user = u;
_container = c;
_in = in;

_eventIdColIndex = DataIteratorUtil.createColumnNameMap(in).get(EVENT_ID_COL);
}

@Override
public int getColumnCount()
{
return _in.getColumnCount();
}

@Override
public ColumnInfo getColumnInfo(int i)
{
return _in.getColumnInfo(i);
}

@Override
public boolean next() throws BatchValidationException
{
boolean hasNext = _in.next();
if (hasNext)
{
Integer eventId = (Integer)_in.get(_eventIdColIndex);
_eventIds.add(eventId);
}
else
{
UserSchema schema = SNDManager.getSndUserSchema(_container, _user);

// Add a post commit task to update the narrative cache after the transaction updating the notes is committed.
SNDManager.get().getTableInfo(schema, "EventNotes").getSchema().getScope().addCommitTask(() -> {
_sndManager.updateNarrativeCache(_container, _user, _eventIds, log);
}, DbScope.CommitTaskOption.POSTCOMMIT);
}
return hasNext;
}

@Override
public Object get(int i)
{
return _in.get(i);
}

@Override
public void close() throws IOException
{
_in.close();
}
}
33 changes: 33 additions & 0 deletions src/org/labkey/snd/query/EventNotesDataIteratorBuilder.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package org.labkey.snd.query;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.data.Container;
import org.labkey.api.dataiterator.DataIterator;
import org.labkey.api.dataiterator.DataIteratorBuilder;
import org.labkey.api.dataiterator.DataIteratorContext;
import org.labkey.api.dataiterator.DataIteratorUtil;
import org.labkey.api.security.User;

public class EventNotesDataIteratorBuilder implements DataIteratorBuilder
{
private final DataIteratorBuilder in;
private final User user;
private final Container container;

public EventNotesDataIteratorBuilder(@NotNull DataIteratorBuilder in, User user, Container container)
{
this.in = in;
this.user = user;
this.container = container;
}

@Override
public DataIterator getDataIterator(DataIteratorContext context)
{
DataIterator it = in.getDataIterator(context);
DataIterator in = DataIteratorUtil.wrapMap(it, false);
return EventNotesDataIterator.wrap(in, context, container, user);
}
}


16 changes: 14 additions & 2 deletions src/org/labkey/snd/query/EventNotesTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.labkey.snd.security.permissions.SNDViewerPermission;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -67,6 +68,7 @@ public UpdateService(SimpleUserSchema.SimpleTable ti)
}

private final SNDService _sndService = SNDService.get();
private final SNDManager _sndManager = SNDManager.get();

private int getRowCount(DataIteratorBuilder rows, @Nullable Map<Enum,Object> configParameters, BatchValidationException errors)
{
Expand Down Expand Up @@ -94,7 +96,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B
{
Logger log = SNDManager.getLogger(configParameters, EventNotesTable.class);
// Large merge triggers importRows path
int result = 0;
int result;
if (getRowCount(rows, configParameters, errors) > SNDManager.MAX_MERGE_ROWS)
{
log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method.");
Expand All @@ -103,11 +105,21 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B
else
{
log.info("Merging rows.");
result = super.mergeRows(user, container, rows, errors, configParameters, extraScriptContext);
DataIteratorBuilder dib = new EventNotesDataIteratorBuilder(rows, user, container);

result = super.mergeRows(user, container, dib, errors, configParameters, extraScriptContext);
}
return result;
}

@Override
public void configureDataIteratorContext(DataIteratorContext context)
{
if (context.getInsertOption() == QueryUpdateService.InsertOption.MERGE)
{
context.addAlternateKeys(Collections.singleton("EventId"));
}
}
}

@Override
Expand Down

0 comments on commit c7708e4

Please sign in to comment.