Skip to content

Commit

Permalink
return async response, write to log instead #3589
Browse files Browse the repository at this point in the history
  • Loading branch information
pdurbin committed Feb 7, 2017
1 parent 50be9f3 commit cc4158c
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 13 deletions.
25 changes: 18 additions & 7 deletions src/main/java/edu/harvard/iq/dataverse/api/Admin.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package edu.harvard.iq.dataverse.api;


import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DvObject;
import edu.harvard.iq.dataverse.EMailValidator;
Expand Down Expand Up @@ -53,8 +54,10 @@
import edu.harvard.iq.dataverse.authorization.RoleAssignee;
import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.ingest.IngestUtil;
import javax.json.JsonArray;
import edu.harvard.iq.dataverse.ingest.IngestAsync;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.Future;
/**
* Where the secure, setup API calls live.
* @author michael
Expand Down Expand Up @@ -868,11 +871,19 @@ public Response findRoleAssignee(@PathParam("idtf") String idtf) {
@Path("datasets/integrity")
@GET
public Response checkDatasetIntegrity() {
JsonArray datasetVersionsWithWrongUnfValue = IngestUtil.getVersionsWithMissingUNFs(datasetSvc.findAll()).build();
JsonObjectBuilder info = Json.createObjectBuilder();
info.add("numProblems", datasetVersionsWithWrongUnfValue.size());
info.add("problems", datasetVersionsWithWrongUnfValue);
return ok(info);
List<Dataset> datasets = datasetSvc.findAll();
SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
String logTimestamp = logFormatter.format(new Date());
String logFile = "../logs/dataset_integrity_" + logTimestamp + ".txt";
Future<JsonObjectBuilder> datasetVersionsWithMissingUNFs = IngestAsync.getVersionsWithMissingUNFs(datasets, logFile);
/**
* @todo Output this data somewhere?
*/
// JsonObjectBuilder info = Json.createObjectBuilder();
// info.add("numProblems", datasetVersionsWithMissingUNFs.size());
// info.add("problems", datasetVersionsWithMissingUNFs);
String status = "Dataset integrity check has begun. See log at " + logFile;
return ok(status);
}

@Path("datasets/integrity/{datasetVersionId}/fixunf")
Expand Down
23 changes: 23 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestAsync.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package edu.harvard.iq.dataverse.ingest;

import edu.harvard.iq.dataverse.Dataset;
import java.util.List;
import java.util.concurrent.Future;
import javax.ejb.AsyncResult;
import javax.ejb.Asynchronous;
import javax.json.Json;
import javax.json.JsonArray;
import javax.json.JsonObjectBuilder;

public class IngestAsync {

@Asynchronous
public static Future<JsonObjectBuilder> getVersionsWithMissingUNFs(List<Dataset> datasets, String logFile) {
JsonObjectBuilder response = Json.createObjectBuilder();
JsonArray datasetVersionsWithMissingUNFs = IngestUtil.getVersionsWithMissingUNFs(datasets, logFile).build();
response.add("datasetVersionsWithMissingUNFs", datasetVersionsWithMissingUNFs);
return new AsyncResult<>(response);

}

}
16 changes: 14 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,14 @@
import edu.harvard.iq.dataverse.util.FileUtil;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.json.Json;
import javax.json.JsonArrayBuilder;
Expand Down Expand Up @@ -280,32 +281,43 @@ public static boolean shouldHaveUnf(DatasetVersion version) {
// i.e. recalculate on the fly the UNFs for the versions that have them,
// compare the 2, and list the mismatches.

public static JsonArrayBuilder getVersionsWithMissingUNFs(List<Dataset> datasets) {
public static JsonArrayBuilder getVersionsWithMissingUNFs(List<Dataset> datasets, String logFile) {
JsonArrayBuilder datasetVersionsWithMissingUNFs = Json.createArrayBuilder();
PrintWriter datasetIntegrityLog = null;
try {
datasetIntegrityLog = new PrintWriter(new FileWriter(logFile));
} catch (IOException ex) {
logger.info("Problem opening log file for dataset integrity check: " + ex);
return datasetVersionsWithMissingUNFs;
}
if (datasets == null || datasets.isEmpty()) {
return datasetVersionsWithMissingUNFs;
}
for (Dataset dataset : datasets) {
for (DatasetVersion dsv : dataset.getVersions()) {
datasetIntegrityLog.println("Checking integrity of dataset id " + dataset.getId() + ", dataset version id " + dsv.getId());
boolean shouldHaveUnf = IngestUtil.shouldHaveUnf(dsv);
String existingUnf = dsv.getUNF();
if (shouldHaveUnf) {
if (existingUnf == null) {
String msg = "Dataset version " + dsv.getSemanticVersion() + " (datasetVersionId " + dsv.getId() + ") from " + dsv.getDataset().getGlobalId() + " doesn't have a UNF but should!";
datasetIntegrityLog.println(msg);
JsonObjectBuilder problem = Json.createObjectBuilder();
problem.add("datasetVersionId", dsv.getId());
problem.add("message", msg);
datasetVersionsWithMissingUNFs.add(problem);
}
} else if (existingUnf != null) {
String msg = "Dataset version " + dsv.getSemanticVersion() + " (datasetVersionId " + dsv.getId() + ") from " + dsv.getDataset().getGlobalId() + " has a UNF (" + existingUnf + ") but shouldn't!";
datasetIntegrityLog.println(msg);
JsonObjectBuilder problem = Json.createObjectBuilder();
problem.add("datasetVersionId", dsv.getId());
problem.add("message", msg);
datasetVersionsWithMissingUNFs.add(problem);
}
}
}
datasetIntegrityLog.close();
return datasetVersionsWithMissingUNFs;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@

public class IngestUtilTest {

String logFile = "/tmp/testLogFile";

@Test
/**
* Test adding duplicate file name labels to a dataset version with no
Expand Down Expand Up @@ -590,9 +592,9 @@ public void testGetUnfData() {
Dataset dataset = new Dataset();
DatasetVersion dsv1 = new DatasetVersion();
datasets.add(dataset);
IngestUtil.getVersionsWithMissingUNFs(datasets);
IngestUtil.getVersionsWithMissingUNFs(datasets, logFile);
assertEquals(null, dsv1.getUNF());
assertEquals(Json.createArrayBuilder().build(), IngestUtil.getVersionsWithMissingUNFs(null).build());
assertEquals(Json.createArrayBuilder().build(), IngestUtil.getVersionsWithMissingUNFs(null, logFile).build());
}

@Test
Expand Down Expand Up @@ -623,7 +625,7 @@ public void testDatasetShouldNotHaveUnf() {
datasetVersions.add(dsv1);
dataset.setVersions(datasetVersions);
datasets.add(dataset);
JsonArray array = IngestUtil.getVersionsWithMissingUNFs(datasets).build();
JsonArray array = IngestUtil.getVersionsWithMissingUNFs(datasets, logFile).build();
System.out.println("array: " + array);
assertEquals("pretendThisIsValidUnf", dsv1.getUNF());
assertEquals(42, array.getJsonObject(0).getInt("datasetVersionId"));
Expand Down Expand Up @@ -660,7 +662,7 @@ public void testDatasetShouldHaveUnf() {

dataset.setVersions(datasetVersions);
datasets.add(dataset);
JsonArray array = IngestUtil.getVersionsWithMissingUNFs(datasets).build();
JsonArray array = IngestUtil.getVersionsWithMissingUNFs(datasets, logFile).build();
System.out.println("array: " + array);
assertEquals(42, array.getJsonObject(0).getInt("datasetVersionId"));
assertEquals("Dataset version DRAFT (datasetVersionId 42) from doi:fakeAuthority/12345 doesn't have a UNF but should!", array.getJsonObject(0).getString("message"));
Expand Down

0 comments on commit cc4158c

Please sign in to comment.