From c15c74bca5de8372d3bd9dbcd693b3d6f3d9e04f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 27 Aug 2021 15:51:22 -0400 Subject: [PATCH 1/8] #8073 removes filename from response --- .../edu/harvard/iq/dataverse/api/Dataverses.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 5725636d787..0022dc02bf9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -946,10 +946,9 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d @GET @Path("{identifier}/guestbookResponses/") - @Produces({"application/download"}) public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf, @QueryParam("guestbookId") Long gbId, @Context HttpServletResponse response) { - + try { Dataverse dv = findDataverseOrDie(dvIdtf); User u = findUserOrDie(); @@ -960,13 +959,8 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String } else { return error(Status.FORBIDDEN, "Not authorized"); } - - String fileTimestamp = dateFormatter.format(new Date()); - String filename = dv.getAlias() + "_GBResponses_" + fileTimestamp + ".csv"; - - response.setHeader("Content-Disposition", "attachment; filename=" - + filename); - ServletOutputStream outputStream = response.getOutputStream(); + + ServletOutputStream outputStream = response.getOutputStream(); Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); From ed6f4896663deac75f0f52bd508ccfb90647dbea Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 30 Aug 2021 13:35:09 -0400 Subject: [PATCH 2/8] #8073 remove "file" from error response --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 0022dc02bf9..761f2323e76 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -973,7 +973,7 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String } return Response.ok().build(); } catch (IOException io) { - return error(Status.BAD_REQUEST, "Failed to produce response file. Exception: " + io.getMessage()); + return error(Status.BAD_REQUEST, "Failed to produce response. Exception: " + io.getMessage()); } catch (WrappedResponse wr) { return wr.getResponse(); } From 6e2548e1aaef93e84e711d0f8ce038bb71146235 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 1 Sep 2021 10:17:20 -0400 Subject: [PATCH 3/8] #8073 release notes --- doc/release-notes/8073-gb-response_api_update.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/8073-gb-response_api_update.md diff --git a/doc/release-notes/8073-gb-response_api_update.md b/doc/release-notes/8073-gb-response_api_update.md new file mode 100644 index 00000000000..ef238c5f35b --- /dev/null +++ b/doc/release-notes/8073-gb-response_api_update.md @@ -0,0 +1,3 @@ +### Update Retrieve Guestbook Response API to remove default output file + +With this release the Retrieve Guestbook Responses for a Dataverse Collection will no longer produce a file by default. You may specify an output file by adding a -o $YOURFILENAME to the curl command. From 15a2fa6e02cb3d93633084c61663b9ca4c41a40b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 1 Sep 2021 10:27:18 -0400 Subject: [PATCH 4/8] 8073 update gb response api doc --- doc/sphinx-guides/source/api/native-api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 823efe05669..fad2b96270b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -587,7 +587,7 @@ You should expect a 200 ("OK") response and JSON output. Retrieve Guestbook Responses for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included in the file. +In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included in the file. The FILENAME parameter is optional without it the responses will be displayed in the console. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -597,14 +597,15 @@ In order to retrieve a file containing a list of Guestbook Responses in csv form export SERVER_URL=https://demo.dataverse.org export ID=root export GUESTBOOK_ID=1 + export FILENAME=myResponses.csv - curl -O -J -f -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID + curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID -o $FILENAME The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -O -J -f -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 -o myResponses.csv Datasets -------- From 0d5c2253049cb57027c9eb1d986e6bb9b2aaadd8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 1 Sep 2021 10:34:36 -0400 Subject: [PATCH 5/8] 8073 remove reference to default output file from doc --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fad2b96270b..d4230043f71 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -587,7 +587,7 @@ You should expect a 200 ("OK") response and JSON output. Retrieve Guestbook Responses for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included in the file. The FILENAME parameter is optional without it the responses will be displayed in the console. +In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included. The FILENAME parameter is optional without it the responses will be displayed in the console. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. From 3ebc0fd8e0e55e92ea612f76e07220c0113c97b3 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 9 Sep 2021 14:57:55 -0400 Subject: [PATCH 6/8] #8073 modify GB response api to use streaming output --- .../harvard/iq/dataverse/api/Dataverses.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 761f2323e76..4bbdf0951d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -104,15 +104,17 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import java.io.IOException; +import java.io.OutputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; import java.util.Map; import java.util.Optional; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; +import javax.ws.rs.core.StreamingOutput; import javax.xml.stream.XMLStreamException; /** @@ -141,6 +143,9 @@ public class Dataverses extends AbstractApiBean { @EJB GuestbookServiceBean guestbookService; + + @EJB + DataverseServiceBean dataverseService; @POST public Response addRoot(String body) { @@ -960,24 +965,29 @@ public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String return error(Status.FORBIDDEN, "Not authorized"); } - ServletOutputStream outputStream = response.getOutputStream(); - - Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); - - List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); - outputStream.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); - for (Object[] result : guestbookResults) { - StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result); - outputStream.write(sb.toString().getBytes()); - outputStream.flush(); - } - return Response.ok().build(); - } catch (IOException io) { - return error(Status.BAD_REQUEST, "Failed to produce response. Exception: " + io.getMessage()); } catch (WrappedResponse wr) { return wr.getResponse(); } + StreamingOutput stream = new StreamingOutput() { + + @Override + public void write(OutputStream os) throws IOException, + WebApplicationException { + + Dataverse dv = dataverseService.findByAlias(dvIdtf); + Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); + + List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); + os.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); + for (Object[] result : guestbookResults) { + StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result); + os.write(sb.toString().getBytes()); + + } + } + }; + return Response.ok(stream).build(); } @PUT From 7074aca6cde3ec0aeed0b74f0e097ce3d419eb50 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 13 Sep 2021 21:41:13 -0400 Subject: [PATCH 7/8] Added a database query optimization, making the API more streamable for complicated dataverses like covid-rt in prod. (#8073) --- .../iq/dataverse/DataverseServiceBean.java | 16 ++++++ .../GuestbookResponseServiceBean.java | 53 +++++++++++++++++-- .../harvard/iq/dataverse/api/Dataverses.java | 5 +- 3 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 520c3ff14df..e092f209acd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -89,6 +89,12 @@ public class DataverseServiceBean implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; + private static final String BASE_QUERY_DATASET_TITLES_WITHIN_DV = "select v.value, o.id\n" + + "from datasetfieldvalue v, dvobject o " + + "where " + + "v.datasetfield_id = (select id from datasetfield f where datasetfieldtype_id = 1 " + + "and datasetversion_id = (select max(id) from datasetversion where dataset_id = o.id))"; + public Dataverse save(Dataverse dataverse) { dataverse.setModificationTime(new Timestamp(new Date().getTime())); @@ -913,4 +919,14 @@ public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList ro return (result); } + // A quick custom query that finds all the (direct children) dataset titles + // with a dataverse and returns a list of (dataset_id, title) pairs. + public List getDatasetTitlesWithinDataverse(Long dataverseId) { + String cqString = BASE_QUERY_DATASET_TITLES_WITHIN_DV + + "and o.owner_id = " + dataverseId; + + return em.createNativeQuery(cqString).getResultList(); + } + + } diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index f2d290215da..32e6500f822 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.logging.Logger; +import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; @@ -43,11 +44,14 @@ public class GuestbookResponseServiceBean { private static final Logger logger = Logger.getLogger(GuestbookResponseServiceBean.class.getCanonicalName()); + @EJB + DataverseServiceBean dataverseService; + // The query below is used for retrieving guestbook responses used to download // the collected data, in CSV format, from the manage-guestbooks and // guestbook-results pages. (for entire dataverses, and for the individual // guestbooks within dataverses, respectively). -- L.A. - private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, v.value, r.responsetime, f.downloadtype," + /*private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, v.value, r.responsetime, f.downloadtype," + " m.label, r.dataFile_id, r.name, r.email, r.institution, r.position," + " o.protocol, o.authority, o.identifier, d.protocol, d.authority, d.identifier " + "from guestbookresponse r, filedownload f, datasetfieldvalue v, filemetadata m, dvobject o, guestbook g, dvobject d " @@ -59,6 +63,18 @@ public class GuestbookResponseServiceBean { + " and d.id = r.datafile_id " + " and r.id = f.guestbookresponse_id " + " and r.dataset_id = o.id " + + " and r.guestbook_id = g.id ";*/ + + private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, f.downloadtype," + + " m.label, r.dataFile_id, r.name, r.email, r.institution, r.position," + + " o.protocol, o.authority, o.identifier, d.protocol, d.authority, d.identifier " + + "from guestbookresponse r, filedownload f, filemetadata m, dvobject o, guestbook g, dvobject d " + + "where " + + "m.datasetversion_id = (select max(datasetversion_id) from filemetadata where datafile_id =r.datafile_id ) " + + " and m.datafile_id = r.datafile_id " + + " and d.id = r.datafile_id " + + " and r.id = f.guestbookresponse_id " + + " and r.dataset_id = o.id " + " and r.guestbook_id = g.id "; // And this query is used for retrieving guestbook responses for displaying @@ -82,7 +98,7 @@ public class GuestbookResponseServiceBean { + "where q.id = r.customquestion_id " + "and r.guestbookResponse_id = g.id " + "and g.dataset_id = o.id "; - + private static SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("MM/d/yyyy"); @@ -128,12 +144,13 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da // of queries now) -- L.A. Map customQandAs = mapCustomQuestionAnswersAsStrings(dataverseId, guestbookId); + Map datasetTitles = mapDatasetTitles(dataverseId); List guestbookResults = getGuestbookResults( dataverseId, guestbookId ); // the CSV header: out.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); for (Object[] result : guestbookResults) { - StringBuilder sb = convertGuestbookResponsesToCSV(customQandAs, result); + StringBuilder sb = convertGuestbookResponsesToCSV(customQandAs, datasetTitles, result); out.write(sb.toString().getBytes()); out.flush(); } @@ -156,7 +173,7 @@ public List getGuestbookResults(Long dataverseId, Long guestbookId ){ } - public StringBuilder convertGuestbookResponsesToCSV ( Map customQandAs, Object[] result) throws IOException { + public StringBuilder convertGuestbookResponsesToCSV ( Map customQandAs, Map datasetTitles, Object[] result) throws IOException { Integer guestbookResponseId = (Integer)result[0]; @@ -172,7 +189,9 @@ public StringBuilder convertGuestbookResponsesToCSV ( Map custo // Dataset name: - sb.append(((String)result[2]).replace(',', ' ')); + Integer datasetId = (Integer) result[2]; + String datasetTitle = datasetTitles.get(datasetId); + sb.append(datasetTitle == null ? "" : datasetTitle.replace(',', ' ')); sb.append(SEPARATOR); // Dataset persistent identifier: @@ -908,5 +927,29 @@ public List findByAuthenticatedUserId(AuthenticatedUser user) query.setParameter("authenticatedUserId", user.getId()); return query.getResultList(); } + + public Map mapDatasetTitles(Long dataverseId) { + Map ret = new HashMap<>(); + + int count = 0; + + List titleResults = dataverseService.getDatasetTitlesWithinDataverse(dataverseId); + + if (titleResults != null) { + for (Object[] titleObj : titleResults) { + Integer datasetId = (Integer) titleObj[1]; + String datasetTitle = (String) titleObj[0]; + + ret.put(datasetId, datasetTitle); + + count++; + } + } + + logger.info("Found " + count + " dataset titles in dataverse " + dataverseId); + + return ret; + + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 4bbdf0951d2..b2dcc046691 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -977,11 +977,12 @@ public void write(OutputStream os) throws IOException, Dataverse dv = dataverseService.findByAlias(dvIdtf); Map customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId); - + Map datasetTitles = guestbookResponseService.mapDatasetTitles(dv.getId()); + List guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId); os.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes()); for (Object[] result : guestbookResults) { - StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result); + StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, datasetTitles, result); os.write(sb.toString().getBytes()); } From e19bbc06e91b46fb53593bd6ddd86a8a1ba338e2 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 14 Sep 2021 09:46:35 -0400 Subject: [PATCH 8/8] removed some logging from the code. (#8073) --- .../harvard/iq/dataverse/GuestbookResponseServiceBean.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index 32e6500f822..2683eaa410f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -931,8 +931,6 @@ public List findByAuthenticatedUserId(AuthenticatedUser user) public Map mapDatasetTitles(Long dataverseId) { Map ret = new HashMap<>(); - int count = 0; - List titleResults = dataverseService.getDatasetTitlesWithinDataverse(dataverseId); if (titleResults != null) { @@ -942,12 +940,9 @@ public Map mapDatasetTitles(Long dataverseId) { ret.put(datasetId, datasetTitle); - count++; } } - logger.info("Found " + count + " dataset titles in dataverse " + dataverseId); - return ret; }