From 826b6b686b0cfb8509d06ca65cd35f9b38f9cec7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 23 May 2017 16:03:06 -0400 Subject: [PATCH 01/17] Allow download of rsync scripts from Data Capture Module (DCM) #3725 --- .../source/installation/config.rst | 26 ++ .../installation/data-capture-module.rst | 10 + .../source/installation/index.rst | 1 + .../iq/dataverse/EjbDataverseEngine.java | 9 + .../iq/dataverse/api/AbstractApiBean.java | 11 +- .../harvard/iq/dataverse/api/Datasets.java | 28 +- .../DataCaptureModuleServiceBean.java | 84 ++++++ .../DataCaptureModuleUtil.java | 47 ++++ .../ScriptRequestResponse.java | 40 +++ .../UploadRequestResponse.java | 21 ++ .../engine/command/CommandContext.java | 3 + .../impl/RequestRsyncScriptCommand.java | 76 ++++++ .../settings/SettingsServiceBean.java | 7 + .../iq/dataverse/util/SystemConfig.java | 3 + .../harvard/iq/dataverse/api/DatasetsIT.java | 250 +++++++++++++++++- .../DataCaptureModuleServiceBeanIT.java | 73 +++++ .../dataverse/engine/TestCommandContext.java | 5 + 17 files changed, 683 insertions(+), 11 deletions(-) create mode 100644 doc/sphinx-guides/source/installation/data-capture-module.rst create mode 100644 src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datacapturemodule/ScriptRequestResponse.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/datacapturemodule/UploadRequestResponse.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d276a89315a..87360ad09c2 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -722,3 +722,29 @@ Set the base URL for the "Compute" button for a dataset. Set the base URL for the "Compute" button for a dataset. ``curl -X PUT -d 'Massachusetts Open Cloud (MOC)' http://localhost:8080/api/admin/settings/:CloudEnvironmentName`` + +:DataCaptureModuleUrl ++++++++++++++++++++++ + +The URL for your Data Capture Module (DCM) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/data-capture-module . + +``curl -X PUT -d 'https://dcm.example.edu' http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl`` + +:RepositoryStorageAbstractionLayerUrl ++++++++++++++++++++++++++++++++++++++ + +The URL for your Repository Storage Abstraction Layer (RSAL) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/rsal . + +``curl -X PUT -d 'https://rsal.example.edu' http://localhost:8080/api/admin/settings/:RepositoryStorageAbstractionLayerUrl`` + +:UploadMethods +++++++++++++++ + +This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``RSYNC`` it will allow your users to download rsync scripts from the DCM. + +``curl -X PUT -d 'RSYNC' http://localhost:8080/api/admin/settings/:UploadMethods`` + +:DownloadMethods +++++++++++++++++ + +This setting is experimental and related to Repository Storage Abstraction Layer (RSAL). As of this writing it has no effect. diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst new file mode 100644 index 00000000000..bf9ac620c05 --- /dev/null +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -0,0 +1,10 @@ +Data Capture Module +=================== + +Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync. Installation instructions can be found at https://github.com/sbgrid/data-capture-module + +Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting metioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``RSYNC``. This will allow your Dataverse installation to communicate your DCM, so that Dataverse can download rsync scripts for your users. + +As of this writing, the only way to download an rsync script is via API using a URL such as the one below: + +``curl http://localhost:8080/api/datasets/{id}/dataCaptureModule/rsync`` diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 469bb75a481..0f765ea346f 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -21,3 +21,4 @@ Contents: geoconnect shibboleth oauth2 + data-capture-module diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index d8926ea6c82..d2814cf472c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -146,6 +147,9 @@ public class EjbDataverseEngine { @EJB MapLayerMetadataServiceBean mapLayerMetadata; + @EJB + DataCaptureModuleServiceBean dataCaptureModule; + @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; @@ -420,6 +424,11 @@ public MapLayerMetadataServiceBean mapLayerMetadata() { return mapLayerMetadata; } + @Override + public DataCaptureModuleServiceBean dataCaptureModule() { + return dataCaptureModule; + } + }; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index b1d9725da1c..77c9adf9e6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -515,7 +515,16 @@ protected Response ok( boolean value ) { .add("status", STATUS_OK) .add("data", value).build() ).build(); } - + + /** + * @param data Payload to return. + * @param mediaType Non-JSON media type. + * @return Non-JSON response, such as a shell script. + */ + protected Response ok(String data, MediaType mediaType) { + return Response.ok().entity(data).type(mediaType).build(); + } + protected Response created( String uri, JsonObjectBuilder bld ) { return Response.created( URI.create(uri) ) .entity( Json.createObjectBuilder() diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 0e4ca0ea925..5e5f6a523cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -17,8 +17,8 @@ import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; -import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; @@ -43,13 +43,13 @@ import edu.harvard.iq.dataverse.engine.command.impl.ListRoleAssignments; import edu.harvard.iq.dataverse.engine.command.impl.ListVersionsCommand; import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetCommand; +import edu.harvard.iq.dataverse.engine.command.impl.RequestRsyncScriptCommand; import edu.harvard.iq.dataverse.engine.command.impl.SetDatasetCitationDateCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetTargetURLCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetThumbnailCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.export.ExportService; -import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -57,9 +57,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; -import java.io.ByteArrayOutputStream; import java.io.InputStream; -import java.io.OutputStream; import java.io.StringReader; import java.util.Collections; import java.util.List; @@ -115,9 +113,6 @@ public class Datasets extends AbstractApiBean { @EJB MetadataBlockServiceBean metadataBlockService; - @EJB - SettingsServiceBean settingsService; - @EJB DataFileServiceBean fileService; @@ -593,6 +588,25 @@ public Response removeDatasetLogo(@PathParam("id") String idSupplied) { } } + @GET + @Path("{identifier}/dataCaptureModule/rsync") + public Response getRsync(@PathParam("identifier") String id) { + String uploadMethodsSettings = settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods); + if (uploadMethodsSettings == null) { + return error(Response.Status.BAD_REQUEST, SettingsServiceBean.Key.UploadMethods + " is null. This installation of Dataverse is not configured for rsync."); + } + if (!uploadMethodsSettings.contains(SystemConfig.FileUploadMethods.RSYNC.toString())) { + return error(Response.Status.BAD_REQUEST, SettingsServiceBean.Key.UploadMethods + " does not contain " + SystemConfig.FileUploadMethods.RSYNC); + } + try { + Dataset dataset = findDatasetOrDie(id); + ScriptRequestResponse scriptRequestResponse = execCommand(new RequestRsyncScriptCommand(createDataverseRequest(findUserOrDie()), dataset)); + return ok(scriptRequestResponse.getScript(), MediaType.valueOf(MediaType.TEXT_PLAIN)); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + /** * Add a File to an existing Dataset * diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java new file mode 100644 index 00000000000..c28bb96e9d1 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java @@ -0,0 +1,84 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import java.util.logging.Logger; +import com.mashape.unirest.http.HttpResponse; +import com.mashape.unirest.http.JsonNode; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.http.exceptions.UnirestException; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DataCaptureModuleUrl; +import java.io.Serializable; +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.inject.Named; + +/** + * This class contains all the methods that have external runtime dependencies + * such as the Data Capture Module itself and PostgreSQL. + */ +@Stateless +@Named +public class DataCaptureModuleServiceBean implements Serializable { + + private static final Logger logger = Logger.getLogger(DataCaptureModuleServiceBean.class.getCanonicalName()); + /** + * You've got to give the DCM time to cook up an rsync script after making + * an upload request. In dev, 400 milliseconds has been enough. Consider + * making this configurable. + */ + public static long millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls = 500; + + @EJB + SettingsServiceBean settingsService; + + // TODO: Do we care about authenticating to the DCM? If not, no need for AuthenticatedUser here. + public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset) { + String dcmBaseUrl = settingsService.getValueForKey(DataCaptureModuleUrl); + if (dcmBaseUrl == null) { + throw new RuntimeException("Problem POSTing JSON to Data Capture Module. The '" + DataCaptureModuleUrl + "' setting has not been configured."); + } + return makeUploadRequest(dcmBaseUrl, user, dataset); + } + + public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, AuthenticatedUser user, Dataset dataset) { + String uploadRequestUrl = dcmBaseUrl + "/ur.py"; + String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(user, dataset).toString(); + // curl -H 'Content-Type: application/json' -X POST -d '{"datasetId":"42", "userId":"1642","datasetIdentifier":"42"}' http://localhost/ur.py + logger.info("JSON to send to Data Capture Module: " + jsonString); + try { + HttpResponse uploadRequest = Unirest.post(uploadRequestUrl) + .body(jsonString) + .asString(); + UploadRequestResponse uploadRequestResponse = DataCaptureModuleUtil.makeUploadRequest(uploadRequest); + return uploadRequestResponse; + } catch (UnirestException ex) { + logger.info("Error calling " + uploadRequestUrl + ": " + ex); + return null; + } + } + + public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset) { + String dcmBaseUrl = settingsService.getValueForKey(DataCaptureModuleUrl); + if (dcmBaseUrl == null) { + throw new RuntimeException("Problem GETing JSON to Data Capture Module for dataset " + dataset.getId() + " The '" + DataCaptureModuleUrl + "' setting has not been configured."); + } + return getRsyncScriptForDataset(dcmBaseUrl, dataset.getId()); + + } + + public static ScriptRequestResponse getRsyncScriptForDataset(String dcmBaseUrl, long datasetId) { + String scriptRequestUrl = dcmBaseUrl + "/sr.py"; + try { + HttpResponse scriptRequest = Unirest.post(scriptRequestUrl) + .field("datasetIdentifier", datasetId) + .asJson(); + return DataCaptureModuleUtil.getScriptFromRequest(scriptRequest); + } catch (UnirestException ex) { + logger.info("Error calling " + scriptRequestUrl + ": " + ex); + return null; + } + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java new file mode 100644 index 00000000000..b3616faa81b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -0,0 +1,47 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +import com.mashape.unirest.http.HttpResponse; +import com.mashape.unirest.http.JsonNode; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import java.util.logging.Logger; +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; + +public class DataCaptureModuleUtil { + + private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); + + static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { + JsonObjectBuilder jab = Json.createObjectBuilder(); +// // The general rule should be to always pass the user id and dataset id to the DCM. + jab.add("userId", user.getId()); + // FIXME: It would make more sense for the key to be "datasetId" since we're sending the primary key. + jab.add("datasetIdentifier", dataset.getId()); + return jab.build(); + } + + public static ScriptRequestResponse getScriptFromRequest(HttpResponse uploadRequest) { + int status = uploadRequest.getStatus(); + JsonNode body = uploadRequest.getBody(); + logger.info("Got " + status + " with body: " + body); + if (status == 404) { + return new ScriptRequestResponse(status); + } + int httpStatusCode = uploadRequest.getStatus(); + String script = body.getObject().getString("script"); + long datasetId = body.getObject().getLong("datasetIdentifier"); + long userId = body.getObject().getLong("userId"); + ScriptRequestResponse scriptRequestResponse = new ScriptRequestResponse(httpStatusCode, datasetId, userId, script); + return scriptRequestResponse; + } + + static UploadRequestResponse makeUploadRequest(HttpResponse uploadRequest) { + int status = uploadRequest.getStatus(); + String body = uploadRequest.getBody(); + logger.info("Got " + status + " with body: " + body); + return new UploadRequestResponse(uploadRequest.getStatus(), body); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/ScriptRequestResponse.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/ScriptRequestResponse.java new file mode 100644 index 00000000000..1a38113cb34 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/ScriptRequestResponse.java @@ -0,0 +1,40 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +public class ScriptRequestResponse { + + private final int httpStatusCode; + private final long datasetId; + private final long userId; + private final String script; + + public ScriptRequestResponse(int httpStatusCode) { + this.httpStatusCode = httpStatusCode; + this.datasetId = -1; + this.userId = -1; + this.script = null; + } + + public ScriptRequestResponse(int httpStatusCode, long datasetId, long userId, String script) { + this.httpStatusCode = httpStatusCode; + this.datasetId = datasetId; + this.userId = userId; + this.script = script; + } + + public int getHttpStatusCode() { + return httpStatusCode; + } + + public long getDatasetId() { + return datasetId; + } + + public long getUserId() { + return userId; + } + + public String getScript() { + return script; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/UploadRequestResponse.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/UploadRequestResponse.java new file mode 100644 index 00000000000..147246c992a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/UploadRequestResponse.java @@ -0,0 +1,21 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +public class UploadRequestResponse { + + private final int httpStatusCode; + private final String response; + + public UploadRequestResponse(int httpStatusCode, String response) { + this.httpStatusCode = httpStatusCode; + this.response = response; + } + + public int getHttpStatusCode() { + return httpStatusCode; + } + + public String getResponse() { + return response; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index f09ba39f724..dc233a9e254 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -26,6 +26,7 @@ import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -117,4 +118,6 @@ public interface CommandContext { public DatasetVersionServiceBean datasetVersion(); public MapLayerMetadataServiceBean mapLayerMetadata(); + + public DataCaptureModuleServiceBean dataCaptureModule(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java new file mode 100644 index 00000000000..c303dfcac6d --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -0,0 +1,76 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; +import edu.harvard.iq.dataverse.datacapturemodule.UploadRequestResponse; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import java.util.Collections; +import java.util.logging.Logger; + +/** + * Always catch a RuntimeException when calling this command, which may occur on + * any problem contacting the Data Capture Module! We have to throw a + * RuntimeException because otherwise ctxt.engine().submit() will put "OK" for + * "actiontype" in the actionlogrecord rather than "InternalError" if you throw + * a CommandExecutionException. + */ +@RequiredPermissions(Permission.AddDataset) +public class RequestRsyncScriptCommand extends AbstractCommand { + + private static final Logger logger = Logger.getLogger(RequestRsyncScriptCommand.class.getCanonicalName()); + + private final Dataset dataset; + private final DataverseRequest request; + + public RequestRsyncScriptCommand(DataverseRequest requestArg, Dataset datasetArg) { + super(requestArg, datasetArg); + request = requestArg; + dataset = datasetArg; + } + + @Override + public ScriptRequestResponse execute(CommandContext ctxt) throws CommandException { + User user = request.getUser(); + if (!(user instanceof AuthenticatedUser)) { + /** + * @todo get Permission.AddDataset from above somehow rather than + * duplicating it here. + */ + throw new PermissionException("This command can only be called by an AuthenticatedUser, not " + user, + this, Collections.singleton(Permission.AddDataset), dataset); + } + AuthenticatedUser au = (AuthenticatedUser) user; + String errorPreamble = "User id " + au.getId() + " had a problem retrieving rsync script for dataset id " + dataset.getId() + " from Data Capture Module."; + UploadRequestResponse uploadRequestResponse = ctxt.dataCaptureModule().requestRsyncScriptCreation(au, dataset); + int statusCode = uploadRequestResponse.getHttpStatusCode(); + String response = uploadRequestResponse.getResponse(); + if (statusCode != 200) { + throw new RuntimeException("When making the upload request, rather than 200 the status code was " + statusCode + ". The body was \'" + response + "\'. We cannont proceed. Returning."); + } + long millisecondsToSleep = DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls; + logger.info("Message from Data Caputure Module upload request endpoint: " + response + ". Sleeping " + millisecondsToSleep + "milliseconds before making rsync script request."); + try { + Thread.sleep(millisecondsToSleep); + } catch (InterruptedException ex) { + throw new RuntimeException(errorPreamble + "Unable to wait " + millisecondsToSleep + " milliseconds: " + ex.getLocalizedMessage()); + } + + ScriptRequestResponse scriptRequestResponse = ctxt.dataCaptureModule().retreiveRequestedRsyncScript(dataset); + String script = scriptRequestResponse.getScript(); + if (script == null || script.isEmpty()) { + throw new RuntimeException(errorPreamble + "The script was null or empty."); + } + logger.fine("script for dataset " + dataset.getId() + ": " + script); + return scriptRequestResponse; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 0b3ba700bb9..cf5f3c6bf02 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -35,6 +35,13 @@ public class SettingsServiceBean { public enum Key { CloudEnvironmentName, ComputeBaseUrl, + /** + * For example, https://datacapture.example.org + */ + DataCaptureModuleUrl, + RepositoryStorageAbstractionLayerUrl, + UploadMethods, + DownloadMethods, IdentifierGenerationStyle, OAuth2CallbackUrl, DefaultAuthProvider, diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 9b43a38f375..b4cf7871f11 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -643,4 +643,7 @@ public boolean isShibPassiveLoginEnabled() { return settingsService.isTrueForKey(SettingsServiceBean.Key.ShibPassiveLoginEnabled, defaultResponse); } + public enum FileUploadMethods { + RSYNC + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4457c3f61cd..4b8e99dd54c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -1,6 +1,8 @@ package edu.harvard.iq.dataverse.api; import com.jayway.restassured.RestAssured; +import static com.jayway.restassured.RestAssured.given; +import com.jayway.restassured.http.ContentType; import com.jayway.restassured.response.Response; import java.util.logging.Logger; import org.junit.BeforeClass; @@ -24,17 +26,20 @@ import java.util.UUID; import static javax.ws.rs.core.Response.Status.NO_CONTENT; import org.apache.commons.lang.StringUtils; -import static org.hamcrest.CoreMatchers.equalTo; -import org.junit.Assert; -import static com.jayway.restassured.RestAssured.given; import com.jayway.restassured.parsing.Parser; import static com.jayway.restassured.path.json.JsonPath.with; import com.jayway.restassured.path.xml.XmlPath; +import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.ArrayList; import java.util.HashMap; +import javax.json.Json; +import javax.json.JsonObjectBuilder; import static junit.framework.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class DatasetsIT { @@ -51,6 +56,14 @@ public static void setUpClass() { Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport); removeExcludeEmail.then().assertThat() .statusCode(200); + + Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl); + removeDcmUrl.then().assertThat() + .statusCode(200); + + Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); + removeUploadMethods.then().assertThat() + .statusCode(200); } @Test @@ -836,4 +849,235 @@ public void testFileChecksum() { } + /** + * In order for this test to pass you must have the Data Capture Module ( + * https://github.com/sbgrid/data-capture-module ) running. We assume that + * most developers haven't set up the DCM so the test is disabled. + */ + @Test + public void testCreateDatasetWithDcmDependency() { + + boolean disabled = true; + + if (disabled) { + return; + } + + // TODO: Test this with a value like "junk" rather than a valid URL which would give `java.net.MalformedURLException: no protocol`. +// Response setDcmUrl = UtilIT.setSetting(SettingsServiceBean.Key.DataCaptureModuleUrl, "http://localhost:8888/ur.py"); + Response setDcmUrl = UtilIT.setSetting(SettingsServiceBean.Key.DataCaptureModuleUrl, "http://localhost:8888"); + setDcmUrl.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response setUploadMethods = UtilIT.setSetting(SettingsServiceBean.Key.UploadMethods, SystemConfig.FileUploadMethods.RSYNC.toString()); + setUploadMethods.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response urlConfigured = given() + // .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/admin/settings/" + SettingsServiceBean.Key.DataCaptureModuleUrl.toString()); + if (urlConfigured.getStatusCode() != 200) { + fail(SettingsServiceBean.Key.DataCaptureModuleUrl + " has not been not configured. This test cannot run without it."); + } + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + long userId = JsonPath.from(createUser.body().asString()).getLong("data.authenticatedUser.id"); + + /** + * @todo Query system to see which file upload mechanisms are available. + */ +// String dataverseAlias = "dv" + UtilIT.getRandomIdentifier(); +// List fileUploadMechanismsEnabled = Arrays.asList(Dataset.FileUploadMechanism.RSYNC.toString()); +// Response createDataverseResponse = UtilIT.createDataverse(dataverseAlias, fileUploadMechanismsEnabled, apiToken); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + // .body("data.alias", equalTo(dataverseAlias)) + // .body("data.fileUploadMechanismsEnabled[0]", equalTo(Dataset.FileUploadMechanism.RSYNC.toString())) + .statusCode(201); + + /** + * @todo Make this configurable at runtime similar to + * UtilIT.getRestAssuredBaseUri + */ +// Response createDatasetResponse = UtilIT.createDatasetWithDcmDependency(dataverseAlias, apiToken); +// Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + Response getDatasetResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/" + datasetId); + getDatasetResponse.prettyPrint(); + getDatasetResponse.then().assertThat() + .statusCode(200); + +// final List> dataTypeField = JsonPath.with(getDatasetResponse.body().asString()) +// .get("data.latestVersion.metadataBlocks.citation.fields.findAll { it.typeName == 'dataType' }"); +// logger.fine("dataTypeField: " + dataTypeField); +// assertThat(dataTypeField.size(), equalTo(1)); +// assertEquals("dataType", dataTypeField.get(0).get("typeName")); +// assertEquals("controlledVocabulary", dataTypeField.get(0).get("typeClass")); +// assertEquals("X-Ray Diffraction", dataTypeField.get(0).get("value")); +// assertTrue(dataTypeField.get(0).get("multiple").equals(false)); + /** + * @todo Also test user who doesn't have permission. + */ + Response getRsyncScriptPermErrorGuest = given() + .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + getRsyncScriptPermErrorGuest.prettyPrint(); + getRsyncScriptPermErrorGuest.then().assertThat() + .contentType(ContentType.JSON) + .body("message", equalTo("User :guest is not permitted to perform requested action.")) + .statusCode(401); + + Response createNoPermsUser = UtilIT.createRandomUser(); + String noPermsUsername = UtilIT.getUsernameFromResponse(createNoPermsUser); + String noPermsApiToken = UtilIT.getApiTokenFromResponse(createNoPermsUser); + + Response getRsyncScriptPermErrorNonGuest = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, noPermsApiToken) + .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + getRsyncScriptPermErrorNonGuest.then().assertThat() + // .statusCode(NO_CONTENT.getStatusCode()); + .contentType(ContentType.JSON) + .body("message", equalTo("User @" + noPermsUsername + " is not permitted to perform requested action.")) + .statusCode(401); + + Response getRsyncScript = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + getRsyncScript.prettyPrint(); + System.out.println("content type: " + getRsyncScript.getContentType()); // text/html;charset=ISO-8859-1 + getRsyncScript.then().assertThat() + .contentType(ContentType.TEXT) + .statusCode(200); + String rsyncScript = getRsyncScript.body().asString(); + System.out.println("script:\n" + rsyncScript); + assertTrue(rsyncScript.startsWith("#!")); + assertTrue(rsyncScript.contains(datasetId.toString())); + + Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); + removeUploadMethods.then().assertThat() + .statusCode(200); + + Response createUser2 = UtilIT.createRandomUser(); + String username2 = UtilIT.getUsernameFromResponse(createUser2); + String apiToken2 = UtilIT.getApiTokenFromResponse(createUser2); + + Response createDataverseResponse2 = UtilIT.createRandomDataverse(apiToken2); + createDataverseResponse2.prettyPrint(); + String dataverseAlias2 = UtilIT.getAliasFromResponse(createDataverseResponse2); + + Response createDatasetResponse2 = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias2, apiToken2); + createDatasetResponse2.prettyPrint(); + Integer datasetId2 = JsonPath.from(createDatasetResponse2.body().asString()).getInt("data.id"); + System.out.println("dataset id: " + datasetId); + + Response attemptToGetRsyncScriptForNonRsyncDataset = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken2) + .get("/api/datasets/" + datasetId2 + "/dataCaptureModule/rsync"); + attemptToGetRsyncScriptForNonRsyncDataset.prettyPrint(); + attemptToGetRsyncScriptForNonRsyncDataset.then().assertThat() + .statusCode(400) + .body("message", equalTo(":UploadMethods is null. This installation of Dataverse is not configured for rsync.")); + + boolean readyToStartWorkingOnChecksumValidationReporting = false; + if (!readyToStartWorkingOnChecksumValidationReporting) { + return; + } + + /** + * Here we are pretending to be the Data Capture Module reporting on if + * checksum validation success or failure. Don't notify the user on + * success (too chatty) but do notify on failure. + * + * @todo On success a process should be kicked off to crawl the files so + * they are imported into Dataverse. Once the crawling and importing is + * complete, notify the user. + * + * @todo What authentication should be used here? The API token of the + * user? (If so, pass the token in the initial upload request payload.) + * This is suboptimal because of the security risk of having the Data + * Capture Module store the API token. Or should Dataverse be able to be + * configured so that it only will receive these messages from trusted + * IP addresses? Should there be a shared secret that's used for *all* + * requests from the Data Capture Module to Dataverse? + */ + JsonObjectBuilder wrongDataset = Json.createObjectBuilder(); + wrongDataset.add("userId", userId); + wrongDataset.add("datasetId", datasetId2); + wrongDataset.add("status", "validation passed"); + Response datasetWithoutRsyncScript = given() + .body(wrongDataset.build().toString()) + .contentType(ContentType.JSON) + .post("/api/datasets/" + datasetId2 + "/dataCaptureModule/checksumValidation"); + datasetWithoutRsyncScript.prettyPrint(); + + datasetWithoutRsyncScript.then().assertThat() + .statusCode(400) + .body("message", equalTo("Dataset id " + datasetId2 + " does not have an rsync script.")); + + JsonObjectBuilder badNews = Json.createObjectBuilder(); + badNews.add("userId", userId); + badNews.add("datasetId", datasetId); + // Status options are documented at https://github.com/sbgrid/data-capture-module/blob/master/doc/api.md#post-upload + badNews.add("status", "validation failed"); + Response uploadFailed = given() + .body(badNews.build().toString()) + .contentType(ContentType.JSON) + .post("/api/datasets/" + datasetId + "/dataCaptureModule/checksumValidation"); + uploadFailed.prettyPrint(); + + uploadFailed.then().assertThat() + /** + * @todo Double check that we're ok with 200 here. We're saying + * "Ok, the bad news was delivered." + */ + .statusCode(200) + .body("data.message", equalTo("User notified about checksum validation failure.")); + + /** + * @todo How can we test what the checksum validation notification looks + * like in the GUI? There is no API for retrieving notifications. + * + * @todo How can we test that the email notification looks ok? + */ +// System.out.println("try logging in with " + username); + // Meanwhile, the user trys uploading again... + JsonObjectBuilder goodNews = Json.createObjectBuilder(); + goodNews.add("userId", userId); + goodNews.add("datasetId", datasetId); + goodNews.add("status", "validation passed"); + Response uploadSuccessful = given() + .body(goodNews.build().toString()) + .contentType(ContentType.JSON) + .post("/api/datasets/" + datasetId + "/dataCaptureModule/checksumValidation"); + uploadSuccessful.prettyPrint(); + + uploadSuccessful.then().assertThat() + .statusCode(200) + .body("data.message", startsWith("Next we will write code to kick off crawling and importing of files")); + + Response deleteDatasetResponse = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken); + deleteDatasetResponse.prettyPrint(); + assertEquals(200, deleteDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverseResponse.prettyPrint(); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + deleteUserResponse.prettyPrint(); + assertEquals(200, deleteUserResponse.getStatusCode()); + + UtilIT.deleteUser(noPermsUsername); + + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java new file mode 100644 index 00000000000..1c87156b3d6 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java @@ -0,0 +1,73 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeAuthenticatedUser; +import static java.lang.Thread.sleep; +import java.util.Calendar; +import java.util.TimeZone; +import java.util.logging.Logger; +import junit.framework.Assert; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public class DataCaptureModuleServiceBeanIT { + + private static final Logger logger = Logger.getLogger(DataCaptureModuleServiceBeanIT.class.getCanonicalName()); + + @Test + public void testUploadRequestWorking() { + AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); + Dataset dataset = new Dataset(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + long timeInMillis = calendar.getTimeInMillis(); + dataset.setId(timeInMillis); + UploadRequestResponse uploadRequestResponse = DataCaptureModuleServiceBean.makeUploadRequest("http://localhost:8888", user, dataset); + assertEquals(200, uploadRequestResponse.getHttpStatusCode()); + assertTrue(uploadRequestResponse.getResponse().contains("recieved")); + assertEquals("\nrecieved\n", uploadRequestResponse.getResponse()); + } + + @Test + public void testScriptRequestWorking() { + long expectedToWork = 3813; + ScriptRequestResponse scriptRequestResponseGood = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", expectedToWork); + System.out.println("script: " + scriptRequestResponseGood.getScript()); + Assert.assertTrue(scriptRequestResponseGood.getScript().startsWith("#!")); + } + + @Test + public void testScriptRequestNotWorking() { + long notExpectedToWork = Long.MAX_VALUE; + ScriptRequestResponse scriptRequestResponseBad = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", notExpectedToWork); + assertNull(scriptRequestResponseBad.getScript()); + } + + @Test + public void testBothSteps() throws InterruptedException { + // Step 1: Upload request + AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); + Dataset dataset = new Dataset(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + long timeInMillis = calendar.getTimeInMillis(); + dataset.setId(timeInMillis); + UploadRequestResponse uploadRequestResponse = DataCaptureModuleServiceBean.makeUploadRequest("http://localhost:8888", user, dataset); + assertEquals(200, uploadRequestResponse.getHttpStatusCode()); + assertTrue(uploadRequestResponse.getResponse().contains("recieved")); + assertEquals("\nrecieved\n", uploadRequestResponse.getResponse()); + + // If you comment this out, expect to see a 404 when you try to download the script. + sleep(DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls); + + // Step 2: Script request. + ScriptRequestResponse scriptRequestResponseGood = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", dataset.getId()); + System.out.println("script: " + scriptRequestResponseGood.getScript()); + assertNotNull(scriptRequestResponseGood.getScript()); + assertTrue(scriptRequestResponseGood.getScript().startsWith("#!")); + + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index c00c218b37b..d5ff019f6ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -192,4 +193,8 @@ public MapLayerMetadataServiceBean mapLayerMetadata() { return null; } + @Override + public DataCaptureModuleServiceBean dataCaptureModule() { + return null; + } } From c8ef8bb78cc0c0dca5ee0cc0be37db04b8495e27 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 23 May 2017 16:36:47 -0400 Subject: [PATCH 02/17] change from "RSYNC" to "dcm/rsync+ssh" #3725 --- .../source/installation/config.rst | 4 +-- .../installation/data-capture-module.rst | 2 +- .../iq/dataverse/util/SystemConfig.java | 27 ++++++++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 87360ad09c2..a46eedebd33 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -740,9 +740,9 @@ The URL for your Repository Storage Abstraction Layer (RSAL) installation. This :UploadMethods ++++++++++++++ -This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``RSYNC`` it will allow your users to download rsync scripts from the DCM. +This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``dcm/rsync+ssh`` it will allow your users to download rsync scripts from the DCM. -``curl -X PUT -d 'RSYNC' http://localhost:8080/api/admin/settings/:UploadMethods`` +``curl -X PUT -d 'dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` :DownloadMethods ++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst index bf9ac620c05..6d5968e27f1 100644 --- a/doc/sphinx-guides/source/installation/data-capture-module.rst +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -3,7 +3,7 @@ Data Capture Module Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync. Installation instructions can be found at https://github.com/sbgrid/data-capture-module -Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting metioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``RSYNC``. This will allow your Dataverse installation to communicate your DCM, so that Dataverse can download rsync scripts for your users. +Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting metioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate your DCM, so that Dataverse can download rsync scripts for your users. As of this writing, the only way to download an rsync script is via API using a URL such as the one below: diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index b4cf7871f11..6258311b8d0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -644,6 +644,31 @@ public boolean isShibPassiveLoginEnabled() { } public enum FileUploadMethods { - RSYNC + + RSYNC("dcm/rsync+ssh"), + NATIVE("NATIVE"); + + private final String text; + + private FileUploadMethods(final String text) { + this.text = text; + } + + public static FileUploadMethods fromString(String text) { + if (text != null) { + for (FileUploadMethods fileUploadMethods : FileUploadMethods.values()) { + if (text.equals(fileUploadMethods.text)) { + return fileUploadMethods; + } + } + } + throw new IllegalArgumentException("FileUploadMethods must be one of these values: " + Arrays.asList(FileUploadMethods.values()) + "."); + } + + @Override + public String toString() { + return text; + } } + } From da2cda451e29e044f45c1f018a034c2e42a95c89 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 24 May 2017 13:45:16 -0400 Subject: [PATCH 03/17] make table of contents consistent #3796 --- doc/sphinx-guides/source/api/client-libraries.rst | 2 ++ doc/sphinx-guides/source/api/native-api.rst | 4 ++-- doc/sphinx-guides/source/api/sword.rst | 7 +++++-- doc/sphinx-guides/source/developers/coding-style.rst | 2 ++ doc/sphinx-guides/source/developers/debugging.rst | 2 ++ doc/sphinx-guides/source/developers/documentation.rst | 9 +++++++++ doc/sphinx-guides/source/developers/geospatial.rst | 4 +++- doc/sphinx-guides/source/developers/intro.rst | 2 ++ doc/sphinx-guides/source/developers/testing.rst | 2 ++ doc/sphinx-guides/source/developers/unf/unf-v3.rst | 2 ++ doc/sphinx-guides/source/developers/unf/unf-v5.rst | 2 ++ doc/sphinx-guides/source/developers/unf/unf-v6.rst | 2 ++ doc/sphinx-guides/source/installation/geoconnect.rst | 2 ++ doc/sphinx-guides/source/installation/intro.rst | 2 ++ doc/sphinx-guides/source/installation/upgrading.rst | 2 ++ doc/sphinx-guides/source/style/foundations.rst | 2 ++ doc/sphinx-guides/source/style/patterns.rst | 4 +++- doc/sphinx-guides/source/user/appendix.rst | 2 ++ .../source/user/data-exploration/tworavens.rst | 2 ++ doc/sphinx-guides/source/user/dataset-management.rst | 2 ++ doc/sphinx-guides/source/user/dataverse-management.rst | 2 ++ doc/sphinx-guides/source/user/find-use-data.rst | 2 ++ doc/sphinx-guides/source/user/tabulardataingest/csv.rst | 2 ++ .../source/user/tabulardataingest/excel.rst | 2 ++ .../source/user/tabulardataingest/ingestprocess.rst | 2 ++ .../source/user/tabulardataingest/rdata.rst | 2 ++ doc/sphinx-guides/source/user/tabulardataingest/spss.rst | 2 ++ .../source/user/tabulardataingest/stata.rst | 2 ++ .../source/user/tabulardataingest/supportedformats.rst | 2 ++ 29 files changed, 70 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 21a6c384d35..d6864c29d7e 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -1,6 +1,8 @@ Client Libraries ================ +.. contents:: :local: + Currently there are client libraries for Python, R, and Java that can be used to develop against Dataverse APIs. We use the term "client library" on this page but "Dataverse SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d41508d309c..5edb2d3d60d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,6 +1,8 @@ Native API ========== +.. contents:: :local: + Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some API calls do not require authentication. Calls that do require authentication require the user's API key. That key can be passed either via an extra query parameter, ``key``, as in ``ENPOINT?key=API_KEY``, or via the HTTP header ``X-Dataverse-key``. Note that while the header option normally requires more work on client side, it is considered safer, as the API key is not logged in the server access logs. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts runing in web browsers. These endpoints are marked with a *CORS* badge. @@ -9,8 +11,6 @@ Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some A .. warning:: Dataverse 4.0's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address//api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. -.. contents:: - Endpoints --------- diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index dbc4283395c..192fcbd56fe 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -1,6 +1,11 @@ SWORD API ========= +.. contents:: :local: + +About +----- + SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "profile" of AtomPub (`RFC 5023`_) which is a RESTful API that allows non-Dataverse software to deposit files and metadata into a Dataverse installation. :ref:`client-libraries` are available in Python, Java, R, Ruby, and PHP. Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. @@ -17,8 +22,6 @@ As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datas .. _SWORDv2 specification: http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html -.. contents:: - .. _incompatible: Backward incompatible changes diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 3334d927891..927c8fdec1c 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,6 +2,8 @@ Coding Style ============ +.. contents:: :local: + Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. A lot of it isn't written down, but a draft has been started at https://docs.google.com/document/d/1KTd3FpM1BI3HlBofaZjMmBiQEJtFf11jiiGpQeJzy7A/edit?usp=sharing diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index ea526778cda..5efc72e8149 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -2,6 +2,8 @@ Debugging ========= +.. contents:: :local: + Logging ------- diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 1908dc59ba5..d15f2fbefac 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -2,6 +2,8 @@ Documentation ============= +.. contents:: :local: + Quick Fix ----------- @@ -61,3 +63,10 @@ After sphinx is done processing the files you should notice that the html folder You can click on the files in the html folder to preview the changes. Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the dataverse repository. + +Table of Contents +----------------- + +Every non-index page should have ``.. contents:: :local:`` near the top as decided in https://github.com/IQSS/dataverse/issues/3796 . Here's a one-liner to check if it's missing: + +``ack -L '.. contents:: :local:' --type=rst | grep -v 'index.rst$'`` diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 55fbf9e6808..d968aaa8507 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -2,6 +2,8 @@ Geospatial Data =============== +.. contents:: :local: + How Dataverse Ingests Shapefiles -------------------------------- @@ -171,4 +173,4 @@ The ``get_join_targets()`` function in ``dataverse_layer_services.py`` uses the Saving Join Target Information to Geoconnect Database ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``get_latest_jointarget_information()`` in ``utils.py`` retrieves recent JoinTarget Information from the database. (See the `utils code in GitHub `_.) \ No newline at end of file +The ``get_latest_jointarget_information()`` in ``utils.py`` retrieves recent JoinTarget Information from the database. (See the `utils code in GitHub `_.) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index fa25d14b67e..8c813b97596 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -2,6 +2,8 @@ Introduction ============ +.. contents:: :local: + Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! Intended Audience diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index a79d2d08570..95c74c87c22 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,6 +2,8 @@ Testing ======= +.. contents:: :local: + Unit Tests ---------- diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst index 2bb6f64a6f2..9503f620b6f 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst @@ -3,6 +3,8 @@ UNF Version 3 =========================== +.. contents:: :local: + Version 3 of the UNF algorithm was used by the Dataverse Network software prior to version 2.0, and was implemented in R code. This algorithm was used on digital objects containing vectors of numbers, vectors of character strings, data sets comprising such vectors, and studies comprising one or more such data sets. The UNF V3 algorithm applied to the content of a data set or study is as follows: diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 4fb160c20ea..53048f77de5 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -3,6 +3,8 @@ UNF Version 5 ================================ +.. contents:: :local: + **Important Update:** **UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours.** diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index 2a67e5255f6..3c8cd5b6d71 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -3,6 +3,8 @@ UNF Version 6 ================================ +.. contents:: :local: + *(this document is a draft!)* The document is primarily intended for those who are interested in implementing their own UNF Version 6 calculator. We would like to encourage multiple parallel implementations, since that would be a great (the only, really) way to cross-validate UNF signatures calculated for specific sets of data. diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index 5ac5f360f73..09d9fca9e0a 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -1,6 +1,8 @@ Geoconnect ========== +.. contents:: :local: + Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To understand the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index f3a2d226fb0..97793a4582a 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -2,6 +2,8 @@ Introduction ============ +.. contents:: :local: + Welcome! Thanks for installing `Dataverse `_! Quick Links diff --git a/doc/sphinx-guides/source/installation/upgrading.rst b/doc/sphinx-guides/source/installation/upgrading.rst index 4ad14bb67d6..8cfcc133ff8 100644 --- a/doc/sphinx-guides/source/installation/upgrading.rst +++ b/doc/sphinx-guides/source/installation/upgrading.rst @@ -2,6 +2,8 @@ Upgrading ========= +.. contents:: :local: + When upgrading within Dataverse 4.x, you will need to follow the upgrade instructions for each intermediate version. Upgrades always involve deploying the latest war file but may also include running SQL scripts and updating the schema used by Solr. diff --git a/doc/sphinx-guides/source/style/foundations.rst b/doc/sphinx-guides/source/style/foundations.rst index 2899dfec20e..2161ade0921 100755 --- a/doc/sphinx-guides/source/style/foundations.rst +++ b/doc/sphinx-guides/source/style/foundations.rst @@ -1,6 +1,8 @@ Foundations +++++++++++ +.. contents:: :local: + Foundation elements are the very basic building blocks to create a page in Dataverse. Here we will outline how we've applied Bootstrap CSS to our UI, and how the CSS settings in our stylesheet mesh with it. Each section includes links to relevant parts of the official Bootstrap guides and other useful resources, where you can find more detailed documentation. We will also outline other UI resources like FontCustom and Socicon and how they are utilized. diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst index f51ecd35e35..d4187019499 100644 --- a/doc/sphinx-guides/source/style/patterns.rst +++ b/doc/sphinx-guides/source/style/patterns.rst @@ -1,6 +1,8 @@ Patterns ++++++++ +.. contents:: :local: + Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap `__ like tooltips and dropdowns, and AJAX components from `PrimeFaces `__ like datatables and commandlinks. @@ -729,4 +731,4 @@ The modal is styled using the `Modal component `_. diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index c921ab7523d..ebec173c4ca 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -1,6 +1,8 @@ Dataset + File Management +++++++++++++++++++++++++ +.. contents:: :local: + A dataset in Dataverse is a container for your data, documentation, code, and the metadata describing this Dataset. |image1| diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index b3a1d7e95f1..aa8633ab0b4 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -1,6 +1,8 @@ Dataverse Management ++++++++++++++++++++++++++++ +.. contents:: :local: + A dataverse is a container for datasets (research data, code, documentation, and metadata) and other dataverses, which can be setup for individual researchers, departments, journals and organizations. |image1| diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 5395593a2ad..54067b974d1 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -1,6 +1,8 @@ Finding and Using Data +++++++++++++++++++++++ +.. contents:: :local: + Finding Data ============= diff --git a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst index 87369471567..e14d98eb3ba 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst @@ -1,6 +1,8 @@ CSV ++++++ +.. contents:: :local: + Ingest of Comma-Separated Values files as tabular data. ------------------------------------------------------- diff --git a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst index ea51cda1870..8bae757a3f4 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst @@ -1,6 +1,8 @@ Excel +++++++ +.. contents:: :local: + Excel files (**New** in Dataverse 4.0!) Note: only the "new", XLSX Excel files are supported. We are not planning to add support for the old-style, binary XLS files. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 12ae430bd67..3aaeacb9ce1 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -1,6 +1,8 @@ Tabular Data, Representation, Storage and Ingest +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.. contents:: :local: + This section explains the basics of how tabular data is handled in the application and what happens during the ingest process, as the files uploaded by the user are processed and converted into the diff --git a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst index ae2cc6cf7fe..f9c771e6e82 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst @@ -1,6 +1,8 @@ R Data Format +++++++++++++++++++++++++++++ +.. contents:: :local: + Support for R (.RData) files has been introduced in DVN 3.5. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst index 86a7d864851..1f25cf6453e 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst @@ -1,6 +1,8 @@ SPSS +++++++ +.. contents:: :local: + SPSS data files (POR and SAV formats). Supported Versions diff --git a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst index 764bc815a2f..49d8457d4ee 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst @@ -1,6 +1,8 @@ Stata ++++++++ +.. contents:: :local: + Of all the third party statistical software providers, Stata does the best job at documenting the internal format of their files, by far. And at making that documentation freely and easily available to developers (yes, we are looking at you, SPSS). Because of that, Stata is the best supported format for tabular data ingest. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst index b5104a5a2c0..030c8c50183 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst @@ -1,6 +1,8 @@ Supported File Formats +++++++++++++++++++++++++++++ +.. contents:: :local: + Tabular Data ingest supports the following file formats: (see the sections below for more information on each of the supported formats) From e102835122bcf5ec244829040f96b9b2f5ab7d64 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 24 May 2017 15:02:50 -0400 Subject: [PATCH 04/17] indicate that we are running rsync over ssh #3725 --- doc/sphinx-guides/source/installation/data-capture-module.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst index 6d5968e27f1..84bde0a5de4 100644 --- a/doc/sphinx-guides/source/installation/data-capture-module.rst +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -1,7 +1,7 @@ Data Capture Module =================== -Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync. Installation instructions can be found at https://github.com/sbgrid/data-capture-module +Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. Installation instructions can be found at https://github.com/sbgrid/data-capture-module Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting metioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate your DCM, so that Dataverse can download rsync scripts for your users. From 4df9336b6aee6b31adddb8210269bf565e3b189f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 25 May 2017 13:44:28 -0400 Subject: [PATCH 05/17] have CreateDatasetCommand call ur.py, more tests #3725 --- .../harvard/iq/dataverse/api/Datasets.java | 9 +- .../DataCaptureModuleUtil.java | 14 ++- .../command/impl/CreateDatasetCommand.java | 11 +++ .../impl/RequestRsyncScriptCommand.java | 2 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 37 +++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 ++ .../DataCaptureModuleUtilTest.java | 96 +++++++++++++++++++ 7 files changed, 153 insertions(+), 25 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9f0314ce6d4..9bf78370cdc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -18,6 +18,7 @@ import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.dataset.DatasetUtil; @@ -592,12 +593,8 @@ public Response removeDatasetLogo(@PathParam("id") String idSupplied) { @GET @Path("{identifier}/dataCaptureModule/rsync") public Response getRsync(@PathParam("identifier") String id) { - String uploadMethodsSettings = settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods); - if (uploadMethodsSettings == null) { - return error(Response.Status.BAD_REQUEST, SettingsServiceBean.Key.UploadMethods + " is null. This installation of Dataverse is not configured for rsync."); - } - if (!uploadMethodsSettings.contains(SystemConfig.FileUploadMethods.RSYNC.toString())) { - return error(Response.Status.BAD_REQUEST, SettingsServiceBean.Key.UploadMethods + " does not contain " + SystemConfig.FileUploadMethods.RSYNC); + if (!DataCaptureModuleUtil.rsyncSupportEnabled(settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { + return error(Response.Status.METHOD_NOT_ALLOWED, SettingsServiceBean.Key.UploadMethods + " does not contain " + SystemConfig.FileUploadMethods.RSYNC + "."); } try { Dataset dataset = findDatasetOrDie(id); diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index b3616faa81b..b3ad9d3b1cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -4,6 +4,7 @@ import com.mashape.unirest.http.JsonNode; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonObject; @@ -13,6 +14,15 @@ public class DataCaptureModuleUtil { private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); + public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { +// if (uploadMethodsSettings != null && uploadMethodsSettings.contains(SystemConfig.FileUploadMethods.RSYNC.toString())) { + if (uploadMethodsSettings != null && SystemConfig.FileUploadMethods.RSYNC.toString().equals(uploadMethodsSettings)) { + return true; + } else { + return false; + } + } + static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { JsonObjectBuilder jab = Json.createObjectBuilder(); // // The general rule should be to always pass the user id and dataset id to the DCM. @@ -25,7 +35,7 @@ static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset d public static ScriptRequestResponse getScriptFromRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); JsonNode body = uploadRequest.getBody(); - logger.info("Got " + status + " with body: " + body); + logger.fine("Got " + status + " with body: " + body); if (status == 404) { return new ScriptRequestResponse(status); } @@ -40,7 +50,7 @@ public static ScriptRequestResponse getScriptFromRequest(HttpResponse static UploadRequestResponse makeUploadRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); String body = uploadRequest.getBody(); - logger.info("Got " + status + " with body: " + body); + logger.fine("Got " + status + " with body: " + body); return new UploadRequestResponse(uploadRequest.getStatus(), body); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java index d9a15e21d71..bc4a5e049a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java @@ -6,6 +6,8 @@ import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; +import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -188,6 +190,15 @@ by the Dataset page (in CREATE mode), it already has the persistent ctxt.templates().incrementUsageCount(template.getId()); } + if (DataCaptureModuleUtil.rsyncSupportEnabled(ctxt.settings().getValueForKey(SettingsServiceBean.Key.UploadMethods))) { + try { + ScriptRequestResponse scriptRequestResponse = ctxt.engine().submit(new RequestRsyncScriptCommand(getRequest(), savedDataset)); + logger.fine("script: " + scriptRequestResponse.getScript()); + } catch (RuntimeException ex) { + logger.info("Problem getting rsync script: " + ex.getLocalizedMessage()); + } + } + try { /** * @todo Do something with the result. Did it succeed or fail? diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index c303dfcac6d..c7f02b68708 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -57,7 +57,7 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio throw new RuntimeException("When making the upload request, rather than 200 the status code was " + statusCode + ". The body was \'" + response + "\'. We cannont proceed. Returning."); } long millisecondsToSleep = DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls; - logger.info("Message from Data Caputure Module upload request endpoint: " + response + ". Sleeping " + millisecondsToSleep + "milliseconds before making rsync script request."); + logger.fine("Message from Data Caputure Module upload request endpoint: " + response + ". Sleeping " + millisecondsToSleep + " milliseconds before making rsync script request."); try { Thread.sleep(millisecondsToSleep); } catch (InterruptedException ex) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 4b8e99dd54c..5e7814f51bc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -18,6 +18,7 @@ import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED; import edu.harvard.iq.dataverse.DataFile; import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; import edu.harvard.iq.dataverse.authorization.DataverseRole; @@ -910,6 +911,13 @@ public void testCreateDatasetWithDcmDependency() { createDatasetResponse.prettyPrint(); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + Response getDatasetJson = UtilIT.nativeGet(datasetId, apiToken); + getDatasetJson.prettyPrint(); + String protocol1 = JsonPath.from(getDatasetJson.getBody().asString()).getString("data.protocol"); + String authority1 = JsonPath.from(getDatasetJson.getBody().asString()).getString("data.authority"); + String identifier1 = JsonPath.from(getDatasetJson.getBody().asString()).getString("data.identifier"); + String datasetPersistentId = protocol1 + ":" + authority1 + "/" + identifier1; + Response getDatasetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) .get("/api/datasets/" + datasetId); @@ -925,33 +933,30 @@ public void testCreateDatasetWithDcmDependency() { // assertEquals("controlledVocabulary", dataTypeField.get(0).get("typeClass")); // assertEquals("X-Ray Diffraction", dataTypeField.get(0).get("value")); // assertTrue(dataTypeField.get(0).get("multiple").equals(false)); - /** - * @todo Also test user who doesn't have permission. - */ - Response getRsyncScriptPermErrorGuest = given() - .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + String nullTokenToIndicateGuest = null; + Response getRsyncScriptPermErrorGuest = UtilIT.getRsyncScript(datasetPersistentId, nullTokenToIndicateGuest); getRsyncScriptPermErrorGuest.prettyPrint(); getRsyncScriptPermErrorGuest.then().assertThat() .contentType(ContentType.JSON) .body("message", equalTo("User :guest is not permitted to perform requested action.")) - .statusCode(401); + .statusCode(UNAUTHORIZED.getStatusCode()); Response createNoPermsUser = UtilIT.createRandomUser(); String noPermsUsername = UtilIT.getUsernameFromResponse(createNoPermsUser); String noPermsApiToken = UtilIT.getApiTokenFromResponse(createNoPermsUser); - Response getRsyncScriptPermErrorNonGuest = given() - .header(UtilIT.API_TOKEN_HTTP_HEADER, noPermsApiToken) - .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + Response getRsyncScriptPermErrorNonGuest = UtilIT.getRsyncScript(datasetPersistentId, noPermsApiToken); getRsyncScriptPermErrorNonGuest.then().assertThat() - // .statusCode(NO_CONTENT.getStatusCode()); .contentType(ContentType.JSON) .body("message", equalTo("User @" + noPermsUsername + " is not permitted to perform requested action.")) - .statusCode(401); + .statusCode(UNAUTHORIZED.getStatusCode()); - Response getRsyncScript = given() - .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/datasets/" + datasetId + "/dataCaptureModule/rsync"); + boolean stopEarlyToVerifyTheScriptWasCreated = false; + if (stopEarlyToVerifyTheScriptWasCreated) { + logger.info("On the DCM, does /deposit/gen/upload-" + datasetId + ".bash exist? It should! Creating the dataset should be enough to create it."); + return; + } + Response getRsyncScript = UtilIT.getRsyncScript(datasetPersistentId, apiToken); getRsyncScript.prettyPrint(); System.out.println("content type: " + getRsyncScript.getContentType()); // text/html;charset=ISO-8859-1 getRsyncScript.then().assertThat() @@ -984,8 +989,8 @@ public void testCreateDatasetWithDcmDependency() { .get("/api/datasets/" + datasetId2 + "/dataCaptureModule/rsync"); attemptToGetRsyncScriptForNonRsyncDataset.prettyPrint(); attemptToGetRsyncScriptForNonRsyncDataset.then().assertThat() - .statusCode(400) - .body("message", equalTo(":UploadMethods is null. This installation of Dataverse is not configured for rsync.")); + .body("message", equalTo(":UploadMethods does not contain dcm/rsync+ssh.")) + .statusCode(METHOD_NOT_ALLOWED.getStatusCode()); boolean readyToStartWorkingOnChecksumValidationReporting = false; if (!readyToStartWorkingOnChecksumValidationReporting) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index bd3fa357a96..de65c90a525 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -915,6 +915,15 @@ static Response deleteMapFromFile(long fileId, String apiToken) { .delete("/api/files/" + fileId + "/map?key=" + apiToken); } + static Response getRsyncScript(String datasetPersistentId, String apiToken) { + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + return requestSpecification.get("/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId=" + datasetPersistentId); + } + @Test public void testGetFileIdFromSwordStatementWithNoFiles() { String swordStatementWithNoFiles = "\n" diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java new file mode 100644 index 00000000000..84526d1a864 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java @@ -0,0 +1,96 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +import com.mashape.unirest.http.HttpResponse; +import com.mashape.unirest.http.JsonNode; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeAuthenticatedUser; +import java.io.UnsupportedEncodingException; +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import org.apache.http.HttpResponseFactory; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.DefaultHttpResponseFactory; +import org.apache.http.message.BasicStatusLine; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public class DataCaptureModuleUtilTest { + + @Test + public void testRsyncSupportEnabled() { + System.out.println("rsyncSupportEnabled"); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled(null)); + assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("dcm/rsync+ssh")); + // We haven't finalized what the separator will be yet. + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE:dcm/rsync+ssh")); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE,dcm/rsync+ssh")); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE")); + assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("junk")); + } + + @Test + public void testGenerateJsonForUploadRequest() { + System.out.println("generateJsonForUploadRequest"); + AuthenticatedUser user = makeAuthenticatedUser("Ralph", "Rsync"); + Dataset dataset = new Dataset(); + dataset.setId(42l); + JsonObject result = DataCaptureModuleUtil.generateJsonForUploadRequest(user, dataset); + assertEquals(42, result.getInt("datasetIdentifier")); + int userId = result.getInt("userId"); + assertTrue(Integer.MIN_VALUE <= userId && userId <= Integer.MAX_VALUE); + } + + @Test + public void testGetScriptFromRequestOk() throws UnsupportedEncodingException { + System.out.println("getScriptFromRequestOk"); + HttpResponseFactory factory = new DefaultHttpResponseFactory(); + org.apache.http.HttpResponse response = factory.newHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, null), null); + JsonObjectBuilder jab = Json.createObjectBuilder(); + jab.add("userId", 42); + jab.add("datasetIdentifier", 123); + jab.add("script", "#!/bin/sh"); + response.setEntity(new StringEntity(jab.build().toString())); + HttpResponse httpResponse = new HttpResponse<>(response, JsonNode.class); + ScriptRequestResponse result = DataCaptureModuleUtil.getScriptFromRequest(httpResponse); + assertEquals(200, result.getHttpStatusCode()); + assertEquals(123, result.getDatasetId()); + assertEquals(42, result.getUserId()); + assertEquals("#!/bin/sh", result.getScript()); + } + + @Test + public void testGetScriptFromRequestNotFound() throws UnsupportedEncodingException { + System.out.println("getScriptFromRequestNotFound"); + HttpResponseFactory factory = new DefaultHttpResponseFactory(); + org.apache.http.HttpResponse response = factory.newHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_NOT_FOUND, null), null); + JsonObjectBuilder jab = Json.createObjectBuilder(); + jab.add("userId", 42); + jab.add("datasetIdentifier", 123); + jab.add("script", "#!/bin/sh"); + response.setEntity(new StringEntity(jab.build().toString())); + HttpResponse httpResponse = new HttpResponse<>(response, JsonNode.class); + ScriptRequestResponse result = DataCaptureModuleUtil.getScriptFromRequest(httpResponse); + assertEquals(404, result.getHttpStatusCode()); + assertEquals(-1, result.getDatasetId()); + assertEquals(-1, result.getUserId()); + assertEquals(null, result.getScript()); + } + + @Test + public void testMakeUploadRequest() throws UnsupportedEncodingException { + System.out.println("makeUploadRequest"); + HttpResponseFactory factory = new DefaultHttpResponseFactory(); + org.apache.http.HttpResponse response = factory.newHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, null), null); + response.setEntity(new StringEntity("received")); + HttpResponse httpResponse = new HttpResponse<>(response, String.class); + UploadRequestResponse result = DataCaptureModuleUtil.makeUploadRequest(httpResponse); + assertEquals(200, result.getHttpStatusCode()); + assertEquals("received", result.getResponse()); + } + +} From a00870797798c96194398a38d81a3f10b7c4ae47 Mon Sep 17 00:00:00 2001 From: Derek Murphy Date: Thu, 25 May 2017 16:45:30 -0400 Subject: [PATCH 06/17] Labeled and repositions ToCs, etc [ref: #3796] Added labels to tables of contents and repositioned them below intro blurbs. Added info on how to do this to Developer Documentation guide. Made assorted small formatting changes for consistency. --- .../source/admin/geoconnect-worldmap.rst | 5 +++-- .../source/admin/harvestclients.rst | 5 +++-- .../source/admin/harvestserver.rst | 3 ++- doc/sphinx-guides/source/admin/index.rst | 2 +- .../source/admin/metadataexport.rst | 3 ++- doc/sphinx-guides/source/admin/timers.rst | 4 ++-- .../source/admin/troubleshooting.rst | 5 +++-- doc/sphinx-guides/source/api/apps.rst | 3 ++- .../source/api/client-libraries.rst | 5 +++-- doc/sphinx-guides/source/api/dataaccess.rst | 5 ++--- doc/sphinx-guides/source/api/index.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 5 +++-- doc/sphinx-guides/source/api/search.rst | 3 ++- doc/sphinx-guides/source/api/sword.rst | 3 ++- .../source/developers/branching-strategy.rst | 3 ++- .../source/developers/coding-style.rst | 3 ++- .../source/developers/debugging.rst | 3 ++- .../source/developers/dev-environment.rst | 3 ++- .../source/developers/documentation.rst | 10 +++++++--- .../source/developers/geospatial.rst | 3 ++- doc/sphinx-guides/source/developers/index.rst | 2 +- doc/sphinx-guides/source/developers/intro.rst | 5 +++-- .../source/developers/making-releases.rst | 3 ++- .../source/developers/selinux.rst | 3 ++- .../source/developers/testing.rst | 3 ++- doc/sphinx-guides/source/developers/tools.rst | 3 ++- .../source/developers/unf/index.rst | 18 +++++++++--------- .../source/developers/unf/unf-v3.rst | 3 ++- .../source/developers/unf/unf-v5.rst | 9 +++++---- .../source/developers/unf/unf-v6.rst | 5 ++++- .../source/installation/administration.rst | 19 ++++++++++--------- .../source/installation/config.rst | 3 ++- .../source/installation/geoconnect.rst | 3 ++- .../source/installation/index.rst | 2 +- .../source/installation/installation-main.rst | 3 ++- .../source/installation/intro.rst | 5 +++-- .../source/installation/oauth2.rst | 3 ++- .../source/installation/prep.rst | 3 ++- .../source/installation/prerequisites.rst | 3 ++- .../installation/r-rapache-tworavens.rst | 8 ++++---- .../source/installation/shibboleth.rst | 5 +++-- .../source/installation/upgrading.rst | 3 ++- .../source/style/foundations.rst | 4 ++-- doc/sphinx-guides/source/style/index.rst | 2 +- doc/sphinx-guides/source/style/patterns.rst | 4 ++-- doc/sphinx-guides/source/user/account.rst | 3 ++- doc/sphinx-guides/source/user/appendix.rst | 5 +++-- .../user/data-exploration/tworavens.rst | 6 ++++-- .../source/user/data-exploration/worldmap.rst | 3 ++- .../source/user/dataset-management.rst | 6 ++++-- .../source/user/dataverse-management.rst | 5 +++-- .../source/user/find-use-data.rst | 3 ++- doc/sphinx-guides/source/user/index.rst | 2 +- .../source/user/tabulardataingest/csv.rst | 3 ++- .../source/user/tabulardataingest/excel.rst | 3 ++- .../user/tabulardataingest/ingestprocess.rst | 4 ++-- .../source/user/tabulardataingest/rdata.rst | 4 ++-- .../source/user/tabulardataingest/spss.rst | 5 +++-- .../source/user/tabulardataingest/stata.rst | 3 ++- .../tabulardataingest/supportedformats.rst | 7 +++++-- 60 files changed, 159 insertions(+), 105 deletions(-) diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst index b12f0b26fe8..71220bd7bbf 100644 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst @@ -1,10 +1,11 @@ Geoconnect and WorldMap ======================= -.. contents:: :local: - One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. +.. contents:: On this page: + :local: + Update "mapitlink" ------------------ diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 3b7be95f790..2204b72d65b 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -1,8 +1,9 @@ Managing Harvesting Clients =========================== -.. contents:: :local: - +.. contents:: On this page: + :local: + Your Dataverse as a Metadata Harvester -------------------------------------- diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 333ae27e925..1d83d62336a 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -1,7 +1,8 @@ Managing Harvesting Server and Sets =================================== -.. contents:: :local: +.. contents:: On this page: + :local: Your Dataverse as an OAI server ------------------------------- diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index 5224e9e2a19..e632192aed8 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -10,7 +10,7 @@ This guide documents the functionality only available to the Dataverse Admin ("N These "superuser" tasks are managed via the new page called the Dashboard. A user logged in as a Dataverse Admin will see the Dashboard link rendered in the upper right corner of every Dataverse page. -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index a8931b87716..e3fd4545be1 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -1,7 +1,8 @@ Metadata Export =============== -.. contents:: :local: +.. contents:: On this page: + :local: Automatic Exports ----------------- diff --git a/doc/sphinx-guides/source/admin/timers.rst b/doc/sphinx-guides/source/admin/timers.rst index 1a5c8874661..bb9db45ce08 100644 --- a/doc/sphinx-guides/source/admin/timers.rst +++ b/doc/sphinx-guides/source/admin/timers.rst @@ -3,10 +3,10 @@ Dataverse Application Timers ============================ -.. contents:: :local: - Dataverse uses timers to automatically run scheduled Harvest and Metadata export jobs. +.. contents:: On this page: + :local: Dedicated timer server in a Dataverse server cluster ---------------------------------------------------- diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 792d1067c56..455cb36a57d 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -3,10 +3,11 @@ Troubleshooting =============== -.. contents:: :local: - This new (as of v.4.6) section of the Admin guide is for tips on how to diagnose and fix system problems. +.. contents:: On this page: + :local: + Deployment fails, "EJB Timer Service not available" --------------------------------------------------- diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index bec5a4c9abf..6c6e25b803e 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -5,7 +5,8 @@ The introduction of Dataverse APIs has fostered the development of apps that are The apps below are open source, demonstrating how to use Dataverse APIs. Some of these apps (and others) are built on :doc:`/api/client-libraries` that are available for Dataverse APIs. -.. contents:: :local: +.. contents:: On this page: + :local: Javascript ---------- diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index d6864c29d7e..a58bf0ef6f5 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -1,12 +1,13 @@ Client Libraries ================ -.. contents:: :local: - Currently there are client libraries for Python, R, and Java that can be used to develop against Dataverse APIs. We use the term "client library" on this page but "Dataverse SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. +.. contents:: On this page: + :local: + Python ------ diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 66ea551d446..879f1ef9a44 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -1,12 +1,11 @@ Data Access API =============== -.. contents:: :local: - - The Data Access API provides programmatic download access to the files stored under Dataverse. More advanced features of the Access API include format-specific transformations (thumbnail generation/resizing for images; converting tabular data into alternative file formats) and access to the data-level metadata that describes the contents of the tabular files. +.. contents:: On this page: + :local: Basic File Access ----------------- diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst index fad6c4c00b1..75ea664d86a 100755 --- a/doc/sphinx-guides/source/api/index.rst +++ b/doc/sphinx-guides/source/api/index.rst @@ -15,7 +15,7 @@ Rather than using a production installation of Dataverse, API users are welcome Please note that the APIs in this guide are shipped with the Dataverse software itself but additional APIs are available if you install the "miniverse" application from https://github.com/IQSS/miniverse and give it read only access to your production Dataverse database. http://dataverse.org/metrics is powered by miniverse. -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5edb2d3d60d..081c0991a08 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,8 +1,6 @@ Native API ========== -.. contents:: :local: - Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some API calls do not require authentication. Calls that do require authentication require the user's API key. That key can be passed either via an extra query parameter, ``key``, as in ``ENPOINT?key=API_KEY``, or via the HTTP header ``X-Dataverse-key``. Note that while the header option normally requires more work on client side, it is considered safer, as the API key is not logged in the server access logs. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts runing in web browsers. These endpoints are marked with a *CORS* badge. @@ -11,6 +9,9 @@ Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some A .. warning:: Dataverse 4.0's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address//api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. contents:: On this page: + :local: + Endpoints --------- diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 5ea6ff7a8c4..277718b5b9b 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -1,7 +1,8 @@ Search API ========== -.. contents:: :local: +.. contents:: On this page: + :local: About ----- diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index 192fcbd56fe..b70c8ef123c 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -1,7 +1,8 @@ SWORD API ========= -.. contents:: :local: +.. contents:: On this page: + :local: About ----- diff --git a/doc/sphinx-guides/source/developers/branching-strategy.rst b/doc/sphinx-guides/source/developers/branching-strategy.rst index 05e0134a770..aaa49f1e00e 100755 --- a/doc/sphinx-guides/source/developers/branching-strategy.rst +++ b/doc/sphinx-guides/source/developers/branching-strategy.rst @@ -2,7 +2,8 @@ Branching Strategy ================== -.. contents:: :local: +.. contents:: On this page: + :local: Goals ----- diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 927c8fdec1c..08e806cfdec 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,7 +2,8 @@ Coding Style ============ -.. contents:: :local: +.. contents:: On this page: + :local: Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index 5efc72e8149..32f1d5dc400 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -2,7 +2,8 @@ Debugging ========= -.. contents:: :local: +.. contents:: On this page: + :local: Logging ------- diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 53317e4c459..46d75661c4f 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -2,7 +2,8 @@ Development Environment ======================= -.. contents:: :local: +.. contents:: On this page: + :local: Assumptions ----------- diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index d15f2fbefac..796cddbb91c 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -2,7 +2,8 @@ Documentation ============= -.. contents:: :local: +.. contents:: On this page: + :local: Quick Fix ----------- @@ -67,6 +68,9 @@ Now you can make a commit with the changes to your own fork in GitHub and submit Table of Contents ----------------- -Every non-index page should have ``.. contents:: :local:`` near the top as decided in https://github.com/IQSS/dataverse/issues/3796 . Here's a one-liner to check if it's missing: +Every non-index page should use the following code to display a table of contents of internal sub-headings: :: -``ack -L '.. contents:: :local:' --type=rst | grep -v 'index.rst$'`` + .. contents:: On this page: + :local: + +This code should be placed below any introductory text/images and directly above the first subheading, much like a Wikipedia page. diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index d968aaa8507..d37fbc32360 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -2,7 +2,8 @@ Geospatial Data =============== -.. contents:: :local: +.. contents:: On this page: + :local: How Dataverse Ingests Shapefiles -------------------------------- diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index f37afbbd173..6530fdc94bb 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -6,7 +6,7 @@ Developer Guide ======================================================= -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 8c813b97596..a8d833ee744 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -2,10 +2,11 @@ Introduction ============ -.. contents:: :local: - Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! +.. contents:: On this page: + :local: + Intended Audience ----------------- diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index d690394cad9..6392f7477df 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -2,7 +2,8 @@ Making Releases =============== -.. contents:: :local: +.. contents:: On this page: + :local: Bump Version Numbers -------------------- diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index 7062a7d1f01..dbd974d27d5 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -2,7 +2,8 @@ SELinux ======= -.. contents:: :local: +.. contents:: On this page: + :local: Introduction ------------ diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 95c74c87c22..912257e695e 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,7 +2,8 @@ Testing ======= -.. contents:: :local: +.. contents:: On this page: + :local: Unit Tests ---------- diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index ab18e8b87f3..5310bcaef89 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -4,7 +4,8 @@ Tools These are handy tools for your :doc:`/developers/dev-environment/`. -.. contents:: :local: +.. contents:: On this page: + :local: Netbeans Connector Chrome Extension +++++++++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index dc2f37d0ba9..589c6870f9d 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -4,15 +4,6 @@ Universal Numerical Fingerprint (UNF) ===================================== -Contents: - -.. toctree:: - :maxdepth: 2 - - unf-v3 - unf-v5 - unf-v6 - .. figure:: ./img/unf-diagram.png :align: center :alt: alternate text @@ -40,3 +31,12 @@ Learn more: Micah Altman, Jeff Gill and Michael McDonald, 2003, `Numerical Issues in Statistical Computing for the Social Scientist `_, New York: John Wiley. + +**Contents:** + +.. toctree:: + :maxdepth: 2 + + unf-v3 + unf-v5 + unf-v6 \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst index 9503f620b6f..a1e5d3d5b01 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst @@ -3,7 +3,8 @@ UNF Version 3 =========================== -.. contents:: :local: +.. contents:: On this page: + :local: Version 3 of the UNF algorithm was used by the Dataverse Network software prior to version 2.0, and was implemented in R code. This algorithm was used on digital objects containing vectors of numbers, vectors of character strings, data sets comprising such vectors, and studies comprising one or more such data sets. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 53048f77de5..919b96c6f81 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -3,12 +3,13 @@ UNF Version 5 ================================ -.. contents:: :local: +.. contents:: On this page: + :local: **Important Update:** -**UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours.** +UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. -**To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation.** +To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. -**Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist.** +Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index 3c8cd5b6d71..be23e7772c3 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -3,7 +3,7 @@ UNF Version 6 ================================ -.. contents:: :local: + *(this document is a draft!)* @@ -13,6 +13,9 @@ The document is primarily intended for those who are interested in implementing UNF v5, on which v6 is based, was originally described in Dr. Micah Altman's paper "A Fingerprint Method for Verification of Scientific Data", Springer Verlag, 2008. The reader is encouraged to consult it for the explanation of the theory behind UNF. However, various changes and clarifications concerning the specifics of normalization have been made to the algorithm since the publication. These crucial details were only documented in the author's unpublished edits of the article and in private correspondence. With this document, a serious effort has been made to produce a complete step-by-step description of the entire process. It should be fully sufficient for the purposes of implementing the algorithm. +.. contents:: On this page: + :local: + I. UNF of a Data Vector ------------------------- diff --git a/doc/sphinx-guides/source/installation/administration.rst b/doc/sphinx-guides/source/installation/administration.rst index b08b510c434..55c54c435e2 100644 --- a/doc/sphinx-guides/source/installation/administration.rst +++ b/doc/sphinx-guides/source/installation/administration.rst @@ -3,7 +3,8 @@ Administration This section focuses on system and database administration tasks. Please see the :doc:`/user/index` for tasks having to do with having the "Admin" role on a dataverse or dataset. -.. contents:: :local: +.. contents:: On this page: + :local: Solr Search Index ----------------- @@ -18,15 +19,15 @@ There are two ways to perform a full reindex of the Dataverse search index. Star Clear and Reindex ~~~~~~~~~~~~~~~~~ -Clearing Data from Solr -....................... +**Clearing Data from Solr** +........................... Please note that the moment you issue this command, it will appear to end users looking at the home page that all data is gone! This is because the home page is powered by the search index. ``curl http://localhost:8080/api/admin/index/clear`` -Start Async Reindex -................... +**Start Async Reindex** +....................... Please note that this operation may take hours depending on the amount of data in your system. This known issue is being tracked at https://github.com/IQSS/dataverse/issues/50 @@ -37,13 +38,13 @@ Reindex in Place An alternative to completely clearing the search index is to reindex in place. -Clear Index Timestamps -...................... +**Clear Index Timestamps** +.......................... ``curl -X DELETE http://localhost:8080/api/admin/index/timestamps`` -Start or Continue Async Reindex -................................ +**Start or Continue Async Reindex** +................................... If indexing stops, this command should pick up where it left off based on which index timestamps have been set, which is why we start by clearing these timestamps above. These timestamps are stored in the ``dvobject`` database table. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 368cfaa7c78..aa356205e0a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -8,7 +8,8 @@ Settings within Dataverse itself are managed via JVM options or by manipulating Once you have finished securing and configuring your Dataverse installation, proceed to the :doc:`administration` section. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. -.. contents:: :local: +.. contents:: On this page: + :local: Securing Your Installation -------------------------- diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index 09d9fca9e0a..a3035391462 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -1,7 +1,8 @@ Geoconnect ========== -.. contents:: :local: +.. contents:: On this page: + :local: Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 469bb75a481..fba233a5ffe 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -6,7 +6,7 @@ Installation Guide ================== -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 44768212e34..d06fe222d11 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -4,7 +4,8 @@ Installation Now that the :doc:`prerequisites` are in place, we are ready to execute the Dataverse installation script (the "installer") and verify that the installation was successful by logging in with a "superuser" account. -.. contents:: :local: +.. contents:: On this page: + :local: .. _dataverse-installer: diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index 97793a4582a..fe5e161bb64 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -2,10 +2,11 @@ Introduction ============ -.. contents:: :local: - Welcome! Thanks for installing `Dataverse `_! +.. contents:: On this page: + :local: + Quick Links ----------- diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index 379bebc1daf..df8a10051af 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -1,7 +1,8 @@ OAuth Login: ORCID, GitHub, Google ================================== -.. contents:: :local: +.. contents:: On this page: + :local: Introduction ------------ diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index d6d531cdfaf..729b14d842f 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -8,7 +8,8 @@ Preparation We'll try to get you up and running as quickly as possible, but we thought you might like to hear about your options. :) -.. contents:: :local: +.. contents:: On this page: + :local: Choose Your Own Installation Adventure -------------------------------------- diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 0e909fb1570..5f6dae51480 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -6,7 +6,8 @@ Before running the Dataverse installation script, you must install and configure You **may** find it helpful to look at how the configuration is done automatically by various tools such as Vagrant, Puppet, or Ansible. See the :doc:`prep` section for pointers on diving into these scripts. -.. contents:: :local: +.. contents:: On this page: + :local: Java ---- diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 8b6ca9cf2e2..72b43c8ea3e 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -3,8 +3,6 @@ TwoRavens ========= -.. contents:: :local: - TwoRavens is a web application for tabular data exploration and statistical analysis. It can be integrated with Dataverse, as an **optional** component. While TwoRavens was originally created at IQSS, its developers have since left the organization. Plans for the future of the Dataverse/TwoRavens collaboration are still being worked out. As @@ -39,8 +37,10 @@ rApache and a collection of required third-party R packages. The installation steps for these components are described in the individual sections of the document below. +.. contents:: On this page: + :local: -0. OVERVIEW +0. Overview +++++++++++ TwoRavens is itself a compact JavaScript application that **runs on the user's @@ -93,7 +93,7 @@ section ``1.b.`` where we explain how to completely erase all the previously built packages. -1. PREREQUISITES +1. Prerequisites ++++++++++++++++ a. httpd (Apache): diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 49ba9868f19..cc527ebed88 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -1,8 +1,9 @@ Shibboleth ========== -.. contents:: :local: - +.. contents:: On this page: + :local: + Introduction ------------ diff --git a/doc/sphinx-guides/source/installation/upgrading.rst b/doc/sphinx-guides/source/installation/upgrading.rst index 8cfcc133ff8..b2b75540a64 100644 --- a/doc/sphinx-guides/source/installation/upgrading.rst +++ b/doc/sphinx-guides/source/installation/upgrading.rst @@ -2,7 +2,8 @@ Upgrading ========= -.. contents:: :local: +.. contents:: On this page: + :local: When upgrading within Dataverse 4.x, you will need to follow the upgrade instructions for each intermediate version. diff --git a/doc/sphinx-guides/source/style/foundations.rst b/doc/sphinx-guides/source/style/foundations.rst index 2161ade0921..54512fd58b2 100755 --- a/doc/sphinx-guides/source/style/foundations.rst +++ b/doc/sphinx-guides/source/style/foundations.rst @@ -1,10 +1,10 @@ Foundations +++++++++++ -.. contents:: :local: - Foundation elements are the very basic building blocks to create a page in Dataverse. Here we will outline how we've applied Bootstrap CSS to our UI, and how the CSS settings in our stylesheet mesh with it. Each section includes links to relevant parts of the official Bootstrap guides and other useful resources, where you can find more detailed documentation. We will also outline other UI resources like FontCustom and Socicon and how they are utilized. +.. contents:: On this page: + :local: Grid Layout =========== diff --git a/doc/sphinx-guides/source/style/index.rst b/doc/sphinx-guides/source/style/index.rst index 44ccc9ed2ab..ba6995e1b53 100755 --- a/doc/sphinx-guides/source/style/index.rst +++ b/doc/sphinx-guides/source/style/index.rst @@ -8,7 +8,7 @@ Style Guide This style guide is meant to help developers implement clear and appropriate UI elements consistent with the Dataverse Project's standards. -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst index d4187019499..4682563decc 100644 --- a/doc/sphinx-guides/source/style/patterns.rst +++ b/doc/sphinx-guides/source/style/patterns.rst @@ -1,10 +1,10 @@ Patterns ++++++++ -.. contents:: :local: - Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap `__ like tooltips and dropdowns, and AJAX components from `PrimeFaces `__ like datatables and commandlinks. +.. contents:: On this page: + :local: Navbar ====== diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index f4e92214647..658b2f64b43 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -1,7 +1,8 @@ Account Creation + Management ============================= -.. contents:: :local: +.. contents:: On this page: + :local: Account Information ------------------- diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst index 94abc3bb1e9..2d59106cb60 100755 --- a/doc/sphinx-guides/source/user/appendix.rst +++ b/doc/sphinx-guides/source/user/appendix.rst @@ -3,10 +3,11 @@ Appendix +++++++++ -.. contents:: :local: - Additional documentation complementary to the User Guide. +.. contents:: On this page: + :local: + Metadata References ====================== diff --git a/doc/sphinx-guides/source/user/data-exploration/tworavens.rst b/doc/sphinx-guides/source/user/data-exploration/tworavens.rst index fe3b2cdb17d..09e839d159c 100755 --- a/doc/sphinx-guides/source/user/data-exploration/tworavens.rst +++ b/doc/sphinx-guides/source/user/data-exploration/tworavens.rst @@ -3,11 +3,13 @@ TwoRavens: Tabular Data Exploration +++++++++++++++++++++++++++++++++++ -.. contents:: :local: - **Please note:** This document is a bit old and may need an update. The TwoRavens project has a more recently published user guide on their site: `http://2ra.vn/papers/tworavens-guide.pdf `_. + +.. contents:: On this page: + :local: + Exploring and Analyzing Tabular files in Dataverse ================================================== diff --git a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst index b3a7ea7c815..a82aedd2ece 100644 --- a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst +++ b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst @@ -3,7 +3,8 @@ WorldMap: Geospatial Data Exploration +++++++++++++++++++++++++++++++++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Dataverse and WorldMap ====================== diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index ebec173c4ca..c3a71b378ff 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -1,12 +1,14 @@ Dataset + File Management +++++++++++++++++++++++++ -.. contents:: :local: - A dataset in Dataverse is a container for your data, documentation, code, and the metadata describing this Dataset. |image1| +.. contents:: On this page: + :local: + + Supported Metadata ================== diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index aa8633ab0b4..889690bc384 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -1,8 +1,6 @@ Dataverse Management ++++++++++++++++++++++++++++ -.. contents:: :local: - A dataverse is a container for datasets (research data, code, documentation, and metadata) and other dataverses, which can be setup for individual researchers, departments, journals and organizations. |image1| @@ -11,6 +9,9 @@ Once a user creates a dataverse they, by default, become the administrator of that dataverse. The dataverse administrator has access to manage the settings described in this guide. +.. contents:: On this page: + :local: + Create a Dataverse (Within the "Root" Dataverse) =================================================== diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 54067b974d1..67ad1032745 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -1,7 +1,8 @@ Finding and Using Data +++++++++++++++++++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Finding Data ============= diff --git a/doc/sphinx-guides/source/user/index.rst b/doc/sphinx-guides/source/user/index.rst index 9d231cb5f6d..05973e6fac9 100755 --- a/doc/sphinx-guides/source/user/index.rst +++ b/doc/sphinx-guides/source/user/index.rst @@ -6,7 +6,7 @@ User Guide ================================================= -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst index e14d98eb3ba..c62e7628103 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst @@ -1,7 +1,8 @@ CSV ++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Ingest of Comma-Separated Values files as tabular data. ------------------------------------------------------- diff --git a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst index 8bae757a3f4..af23042bf77 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst @@ -1,7 +1,8 @@ Excel +++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Excel files (**New** in Dataverse 4.0!) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 3aaeacb9ce1..4a9fba56dd6 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -1,13 +1,13 @@ Tabular Data, Representation, Storage and Ingest +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. contents:: :local: - This section explains the basics of how tabular data is handled in the application and what happens during the ingest process, as the files uploaded by the user are processed and converted into the archival format in the Dataverse application. +.. contents:: On this page: + :local: What Happens During this "Ingest"? =================================== diff --git a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst index f9c771e6e82..966b962928f 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst @@ -1,10 +1,10 @@ R Data Format +++++++++++++++++++++++++++++ -.. contents:: :local: - Support for R (.RData) files has been introduced in DVN 3.5. +.. contents:: On this page: + :local: Overview. =========== diff --git a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst index 1f25cf6453e..1ea5f64cde1 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst @@ -1,10 +1,11 @@ SPSS +++++++ -.. contents:: :local: - SPSS data files (POR and SAV formats). +.. contents:: On this page: + :local: + Supported Versions ------------------ diff --git a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst index 49d8457d4ee..5bb389718b5 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst @@ -1,7 +1,8 @@ Stata ++++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Of all the third party statistical software providers, Stata does the best job at documenting the internal format of their files, by far. And at making that documentation freely and easily available to developers (yes, we are looking at you, SPSS). Because of that, Stata is the best supported format for tabular data ingest. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst index 030c8c50183..e2d03ebcf9d 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst @@ -1,10 +1,11 @@ Supported File Formats +++++++++++++++++++++++++++++ -.. contents:: :local: +.. contents:: On this page: + :local: Tabular Data ingest supports the following file formats: -(see the sections below for more information on each of the supported formats) + ================================ ================================== File format Versions supported @@ -15,3 +16,5 @@ R up to 3 Excel XLSX only (XLS is NOT supported) CSV (comma-separated values) (limited support) ================================ ================================== + +See the subsections in the left sidebar for more information on each of these supported formats. \ No newline at end of file From 2739c72663d4562c192e7bed9f167ee41c70170a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 May 2017 09:28:26 -0400 Subject: [PATCH 07/17] use `|toctitle|` as variable from rst_prolog #3796 --- doc/sphinx-guides/source/admin/geoconnect-worldmap.rst | 2 +- doc/sphinx-guides/source/admin/harvestclients.rst | 2 +- doc/sphinx-guides/source/admin/harvestserver.rst | 2 +- doc/sphinx-guides/source/admin/metadataexport.rst | 2 +- doc/sphinx-guides/source/admin/timers.rst | 2 +- doc/sphinx-guides/source/admin/troubleshooting.rst | 2 +- doc/sphinx-guides/source/api/apps.rst | 2 +- doc/sphinx-guides/source/api/client-libraries.rst | 2 +- doc/sphinx-guides/source/api/dataaccess.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 2 +- doc/sphinx-guides/source/api/search.rst | 2 +- doc/sphinx-guides/source/api/sword.rst | 2 +- doc/sphinx-guides/source/conf.py | 4 ++++ doc/sphinx-guides/source/developers/branching-strategy.rst | 2 +- doc/sphinx-guides/source/developers/coding-style.rst | 2 +- doc/sphinx-guides/source/developers/debugging.rst | 2 +- doc/sphinx-guides/source/developers/dev-environment.rst | 2 +- doc/sphinx-guides/source/developers/documentation.rst | 4 ++-- doc/sphinx-guides/source/developers/geospatial.rst | 2 +- doc/sphinx-guides/source/developers/intro.rst | 2 +- doc/sphinx-guides/source/developers/making-releases.rst | 2 +- doc/sphinx-guides/source/developers/selinux.rst | 2 +- doc/sphinx-guides/source/developers/testing.rst | 2 +- doc/sphinx-guides/source/developers/tools.rst | 2 +- doc/sphinx-guides/source/developers/unf/unf-v3.rst | 2 +- doc/sphinx-guides/source/developers/unf/unf-v5.rst | 2 +- doc/sphinx-guides/source/developers/unf/unf-v6.rst | 2 +- doc/sphinx-guides/source/installation/administration.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 2 +- doc/sphinx-guides/source/installation/geoconnect.rst | 2 +- doc/sphinx-guides/source/installation/installation-main.rst | 2 +- doc/sphinx-guides/source/installation/intro.rst | 2 +- doc/sphinx-guides/source/installation/oauth2.rst | 2 +- doc/sphinx-guides/source/installation/prep.rst | 2 +- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- doc/sphinx-guides/source/installation/r-rapache-tworavens.rst | 2 +- doc/sphinx-guides/source/installation/shibboleth.rst | 2 +- doc/sphinx-guides/source/installation/upgrading.rst | 2 +- doc/sphinx-guides/source/style/foundations.rst | 2 +- doc/sphinx-guides/source/style/patterns.rst | 2 +- doc/sphinx-guides/source/user/account.rst | 2 +- doc/sphinx-guides/source/user/appendix.rst | 2 +- doc/sphinx-guides/source/user/data-exploration/tworavens.rst | 2 +- doc/sphinx-guides/source/user/data-exploration/worldmap.rst | 2 +- doc/sphinx-guides/source/user/dataset-management.rst | 2 +- doc/sphinx-guides/source/user/dataverse-management.rst | 2 +- doc/sphinx-guides/source/user/find-use-data.rst | 2 +- doc/sphinx-guides/source/user/tabulardataingest/csv.rst | 2 +- doc/sphinx-guides/source/user/tabulardataingest/excel.rst | 2 +- .../source/user/tabulardataingest/ingestprocess.rst | 2 +- doc/sphinx-guides/source/user/tabulardataingest/rdata.rst | 2 +- doc/sphinx-guides/source/user/tabulardataingest/spss.rst | 2 +- doc/sphinx-guides/source/user/tabulardataingest/stata.rst | 2 +- .../source/user/tabulardataingest/supportedformats.rst | 2 +- 54 files changed, 58 insertions(+), 54 deletions(-) diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst index 71220bd7bbf..0af163a2916 100644 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst @@ -3,7 +3,7 @@ Geoconnect and WorldMap One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. -.. contents:: On this page: +.. contents:: |toctitle| :local: Update "mapitlink" diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 2204b72d65b..25c3d493a38 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -1,7 +1,7 @@ Managing Harvesting Clients =========================== -.. contents:: On this page: +.. contents:: |toctitle| :local: Your Dataverse as a Metadata Harvester diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 1d83d62336a..122ceb0ee28 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -1,7 +1,7 @@ Managing Harvesting Server and Sets =================================== -.. contents:: On this page: +.. contents:: |toctitle| :local: Your Dataverse as an OAI server diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index e3fd4545be1..8c50ceacd84 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -1,7 +1,7 @@ Metadata Export =============== -.. contents:: On this page: +.. contents:: |toctitle| :local: Automatic Exports diff --git a/doc/sphinx-guides/source/admin/timers.rst b/doc/sphinx-guides/source/admin/timers.rst index bb9db45ce08..f118604654b 100644 --- a/doc/sphinx-guides/source/admin/timers.rst +++ b/doc/sphinx-guides/source/admin/timers.rst @@ -5,7 +5,7 @@ Dataverse Application Timers Dataverse uses timers to automatically run scheduled Harvest and Metadata export jobs. -.. contents:: On this page: +.. contents:: |toctitle| :local: Dedicated timer server in a Dataverse server cluster diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 455cb36a57d..fe9e8b7c659 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -5,7 +5,7 @@ Troubleshooting This new (as of v.4.6) section of the Admin guide is for tips on how to diagnose and fix system problems. -.. contents:: On this page: +.. contents:: |toctitle| :local: Deployment fails, "EJB Timer Service not available" diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index 6c6e25b803e..f7c6c75ce0a 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -5,7 +5,7 @@ The introduction of Dataverse APIs has fostered the development of apps that are The apps below are open source, demonstrating how to use Dataverse APIs. Some of these apps (and others) are built on :doc:`/api/client-libraries` that are available for Dataverse APIs. -.. contents:: On this page: +.. contents:: |toctitle| :local: Javascript diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index a58bf0ef6f5..7131093b87e 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -5,7 +5,7 @@ Currently there are client libraries for Python, R, and Java that can be used to Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. -.. contents:: On this page: +.. contents:: |toctitle| :local: Python diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 879f1ef9a44..574cc49616a 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -4,7 +4,7 @@ Data Access API The Data Access API provides programmatic download access to the files stored under Dataverse. More advanced features of the Access API include format-specific transformations (thumbnail generation/resizing for images; converting tabular data into alternative file formats) and access to the data-level metadata that describes the contents of the tabular files. -.. contents:: On this page: +.. contents:: |toctitle| :local: Basic File Access diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 081c0991a08..78d90393d34 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -9,7 +9,7 @@ Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some A .. warning:: Dataverse 4.0's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address//api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. -.. contents:: On this page: +.. contents:: |toctitle| :local: Endpoints diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 277718b5b9b..c41c5d347be 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -1,7 +1,7 @@ Search API ========== -.. contents:: On this page: +.. contents:: |toctitle| :local: About diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index b70c8ef123c..ab2a4e699f0 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -1,7 +1,7 @@ SWORD API ========= -.. contents:: On this page: +.. contents:: |toctitle| :local: About diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index fc9bf11677d..6b3b56ab471 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -427,3 +427,7 @@ intersphinx_mapping = {'http://docs.python.org/': None} # Suppress "WARNING: unknown mimetype for ..." https://github.com/IQSS/dataverse/issues/3391 suppress_warnings = ['epub.unknown_project_files'] +rst_prolog = """ +.. |toctitle| replace:: Contents: +.. |anotherSub| replace:: Yes, there can be multiple. +""" diff --git a/doc/sphinx-guides/source/developers/branching-strategy.rst b/doc/sphinx-guides/source/developers/branching-strategy.rst index aaa49f1e00e..536db58668a 100755 --- a/doc/sphinx-guides/source/developers/branching-strategy.rst +++ b/doc/sphinx-guides/source/developers/branching-strategy.rst @@ -2,7 +2,7 @@ Branching Strategy ================== -.. contents:: On this page: +.. contents:: |toctitle| :local: Goals diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 08e806cfdec..6682a99ddd4 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,7 +2,7 @@ Coding Style ============ -.. contents:: On this page: +.. contents:: |toctitle| :local: Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index 32f1d5dc400..3e69e98a71d 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -2,7 +2,7 @@ Debugging ========= -.. contents:: On this page: +.. contents:: |toctitle| :local: Logging diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 46d75661c4f..09a1fe2ee17 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -2,7 +2,7 @@ Development Environment ======================= -.. contents:: On this page: +.. contents:: |toctitle| :local: Assumptions diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 796cddbb91c..472e44a01b1 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -2,7 +2,7 @@ Documentation ============= -.. contents:: On this page: +.. contents:: |toctitle| :local: Quick Fix @@ -70,7 +70,7 @@ Table of Contents Every non-index page should use the following code to display a table of contents of internal sub-headings: :: - .. contents:: On this page: + .. contents:: |toctitle| :local: This code should be placed below any introductory text/images and directly above the first subheading, much like a Wikipedia page. diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index d37fbc32360..d43e23cbefc 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -2,7 +2,7 @@ Geospatial Data =============== -.. contents:: On this page: +.. contents:: |toctitle| :local: How Dataverse Ingests Shapefiles diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index a8d833ee744..47ad45aaf3c 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -4,7 +4,7 @@ Introduction Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! -.. contents:: On this page: +.. contents:: |toctitle| :local: Intended Audience diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 6392f7477df..bb151d906d0 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -2,7 +2,7 @@ Making Releases =============== -.. contents:: On this page: +.. contents:: |toctitle| :local: Bump Version Numbers diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index dbd974d27d5..c860d123e6a 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -2,7 +2,7 @@ SELinux ======= -.. contents:: On this page: +.. contents:: |toctitle| :local: Introduction diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 912257e695e..763e9f53871 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,7 +2,7 @@ Testing ======= -.. contents:: On this page: +.. contents:: |toctitle| :local: Unit Tests diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index 5310bcaef89..f094fd5fb9b 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -4,7 +4,7 @@ Tools These are handy tools for your :doc:`/developers/dev-environment/`. -.. contents:: On this page: +.. contents:: |toctitle| :local: Netbeans Connector Chrome Extension diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst index a1e5d3d5b01..3f0018d7fa5 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst @@ -3,7 +3,7 @@ UNF Version 3 =========================== -.. contents:: On this page: +.. contents:: |toctitle| :local: Version 3 of the UNF algorithm was used by the Dataverse Network software prior to version 2.0, and was implemented in R code. This algorithm was used on digital objects containing vectors of numbers, vectors of character strings, data sets comprising such vectors, and studies comprising one or more such data sets. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 919b96c6f81..29b4556b1f9 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -3,7 +3,7 @@ UNF Version 5 ================================ -.. contents:: On this page: +.. contents:: |toctitle| :local: **Important Update:** diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index be23e7772c3..b5c898a9a08 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -13,7 +13,7 @@ The document is primarily intended for those who are interested in implementing UNF v5, on which v6 is based, was originally described in Dr. Micah Altman's paper "A Fingerprint Method for Verification of Scientific Data", Springer Verlag, 2008. The reader is encouraged to consult it for the explanation of the theory behind UNF. However, various changes and clarifications concerning the specifics of normalization have been made to the algorithm since the publication. These crucial details were only documented in the author's unpublished edits of the article and in private correspondence. With this document, a serious effort has been made to produce a complete step-by-step description of the entire process. It should be fully sufficient for the purposes of implementing the algorithm. -.. contents:: On this page: +.. contents:: |toctitle| :local: I. UNF of a Data Vector diff --git a/doc/sphinx-guides/source/installation/administration.rst b/doc/sphinx-guides/source/installation/administration.rst index 55c54c435e2..ed1bb0fd775 100644 --- a/doc/sphinx-guides/source/installation/administration.rst +++ b/doc/sphinx-guides/source/installation/administration.rst @@ -3,7 +3,7 @@ Administration This section focuses on system and database administration tasks. Please see the :doc:`/user/index` for tasks having to do with having the "Admin" role on a dataverse or dataset. -.. contents:: On this page: +.. contents:: |toctitle| :local: Solr Search Index diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index aa356205e0a..0f90b335d27 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -8,7 +8,7 @@ Settings within Dataverse itself are managed via JVM options or by manipulating Once you have finished securing and configuring your Dataverse installation, proceed to the :doc:`administration` section. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. -.. contents:: On this page: +.. contents:: |toctitle| :local: Securing Your Installation diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index a3035391462..30d07f3f76c 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -1,7 +1,7 @@ Geoconnect ========== -.. contents:: On this page: +.. contents:: |toctitle| :local: Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index d06fe222d11..a33d7698fab 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -4,7 +4,7 @@ Installation Now that the :doc:`prerequisites` are in place, we are ready to execute the Dataverse installation script (the "installer") and verify that the installation was successful by logging in with a "superuser" account. -.. contents:: On this page: +.. contents:: |toctitle| :local: .. _dataverse-installer: diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index fe5e161bb64..c55e41c98be 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -4,7 +4,7 @@ Introduction Welcome! Thanks for installing `Dataverse `_! -.. contents:: On this page: +.. contents:: |toctitle| :local: Quick Links diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index df8a10051af..d2e18f8b89b 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -1,7 +1,7 @@ OAuth Login: ORCID, GitHub, Google ================================== -.. contents:: On this page: +.. contents:: |toctitle| :local: Introduction diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index 729b14d842f..90a577c1d43 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -8,7 +8,7 @@ Preparation We'll try to get you up and running as quickly as possible, but we thought you might like to hear about your options. :) -.. contents:: On this page: +.. contents:: |toctitle| :local: Choose Your Own Installation Adventure diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 5f6dae51480..a570e5ab84e 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -6,7 +6,7 @@ Before running the Dataverse installation script, you must install and configure You **may** find it helpful to look at how the configuration is done automatically by various tools such as Vagrant, Puppet, or Ansible. See the :doc:`prep` section for pointers on diving into these scripts. -.. contents:: On this page: +.. contents:: |toctitle| :local: Java diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 72b43c8ea3e..4e394461e26 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -37,7 +37,7 @@ rApache and a collection of required third-party R packages. The installation steps for these components are described in the individual sections of the document below. -.. contents:: On this page: +.. contents:: |toctitle| :local: 0. Overview diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index cc527ebed88..59562ba0696 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -1,7 +1,7 @@ Shibboleth ========== -.. contents:: On this page: +.. contents:: |toctitle| :local: Introduction diff --git a/doc/sphinx-guides/source/installation/upgrading.rst b/doc/sphinx-guides/source/installation/upgrading.rst index b2b75540a64..66c491a3e87 100644 --- a/doc/sphinx-guides/source/installation/upgrading.rst +++ b/doc/sphinx-guides/source/installation/upgrading.rst @@ -2,7 +2,7 @@ Upgrading ========= -.. contents:: On this page: +.. contents:: |toctitle| :local: When upgrading within Dataverse 4.x, you will need to follow the upgrade instructions for each intermediate version. diff --git a/doc/sphinx-guides/source/style/foundations.rst b/doc/sphinx-guides/source/style/foundations.rst index 54512fd58b2..d90b617575f 100755 --- a/doc/sphinx-guides/source/style/foundations.rst +++ b/doc/sphinx-guides/source/style/foundations.rst @@ -3,7 +3,7 @@ Foundations Foundation elements are the very basic building blocks to create a page in Dataverse. Here we will outline how we've applied Bootstrap CSS to our UI, and how the CSS settings in our stylesheet mesh with it. Each section includes links to relevant parts of the official Bootstrap guides and other useful resources, where you can find more detailed documentation. We will also outline other UI resources like FontCustom and Socicon and how they are utilized. -.. contents:: On this page: +.. contents:: |toctitle| :local: Grid Layout diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst index 4682563decc..ffd6b607416 100644 --- a/doc/sphinx-guides/source/style/patterns.rst +++ b/doc/sphinx-guides/source/style/patterns.rst @@ -3,7 +3,7 @@ Patterns Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap `__ like tooltips and dropdowns, and AJAX components from `PrimeFaces `__ like datatables and commandlinks. -.. contents:: On this page: +.. contents:: |toctitle| :local: Navbar diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index 658b2f64b43..9e5c9565d1f 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -1,7 +1,7 @@ Account Creation + Management ============================= -.. contents:: On this page: +.. contents:: |toctitle| :local: Account Information diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst index 2d59106cb60..ba38c51fd6f 100755 --- a/doc/sphinx-guides/source/user/appendix.rst +++ b/doc/sphinx-guides/source/user/appendix.rst @@ -5,7 +5,7 @@ Appendix Additional documentation complementary to the User Guide. -.. contents:: On this page: +.. contents:: |toctitle| :local: Metadata References diff --git a/doc/sphinx-guides/source/user/data-exploration/tworavens.rst b/doc/sphinx-guides/source/user/data-exploration/tworavens.rst index 09e839d159c..d6b2311cab0 100755 --- a/doc/sphinx-guides/source/user/data-exploration/tworavens.rst +++ b/doc/sphinx-guides/source/user/data-exploration/tworavens.rst @@ -7,7 +7,7 @@ TwoRavens: Tabular Data Exploration project has a more recently published user guide on their site: `http://2ra.vn/papers/tworavens-guide.pdf `_. -.. contents:: On this page: +.. contents:: |toctitle| :local: Exploring and Analyzing Tabular files in Dataverse diff --git a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst index a82aedd2ece..398c3385da6 100644 --- a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst +++ b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst @@ -3,7 +3,7 @@ WorldMap: Geospatial Data Exploration +++++++++++++++++++++++++++++++++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Dataverse and WorldMap diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index c3a71b378ff..3cc1e052eaf 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -5,7 +5,7 @@ A dataset in Dataverse is a container for your data, documentation, code, and th |image1| -.. contents:: On this page: +.. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index 889690bc384..287f2185ddd 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -9,7 +9,7 @@ Once a user creates a dataverse they, by default, become the administrator of that dataverse. The dataverse administrator has access to manage the settings described in this guide. -.. contents:: On this page: +.. contents:: |toctitle| :local: Create a Dataverse (Within the "Root" Dataverse) diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 67ad1032745..f62877a32fc 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -1,7 +1,7 @@ Finding and Using Data +++++++++++++++++++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Finding Data diff --git a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst index c62e7628103..e30f4889972 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst @@ -1,7 +1,7 @@ CSV ++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Ingest of Comma-Separated Values files as tabular data. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst index af23042bf77..b8612092f06 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst @@ -1,7 +1,7 @@ Excel +++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Excel files (**New** in Dataverse 4.0!) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 4a9fba56dd6..4f71dd9944f 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -6,7 +6,7 @@ the application and what happens during the ingest process, as the files uploaded by the user are processed and converted into the archival format in the Dataverse application. -.. contents:: On this page: +.. contents:: |toctitle| :local: What Happens During this "Ingest"? diff --git a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst index 966b962928f..0707051b5a2 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst @@ -3,7 +3,7 @@ R Data Format Support for R (.RData) files has been introduced in DVN 3.5. -.. contents:: On this page: +.. contents:: |toctitle| :local: Overview. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst index 1ea5f64cde1..71bc61265c3 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst @@ -3,7 +3,7 @@ SPSS SPSS data files (POR and SAV formats). -.. contents:: On this page: +.. contents:: |toctitle| :local: Supported Versions diff --git a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst index 5bb389718b5..e17e7321eae 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst @@ -1,7 +1,7 @@ Stata ++++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Of all the third party statistical software providers, Stata does the best job at documenting the internal format of their files, by far. And at making that documentation freely and easily available to developers (yes, we are looking at you, SPSS). Because of that, Stata is the best supported format for tabular data ingest. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst index e2d03ebcf9d..6078d37ee84 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst @@ -1,7 +1,7 @@ Supported File Formats +++++++++++++++++++++++++++++ -.. contents:: On this page: +.. contents:: |toctitle| :local: Tabular Data ingest supports the following file formats: From 3f6f43010f1af3f1ded46b8246c605e7ff8586f2 Mon Sep 17 00:00:00 2001 From: Derek Murphy Date: Fri, 26 May 2017 12:46:18 -0400 Subject: [PATCH 08/17] Typo fixes Typo fixes to DCM page. --- .../source/installation/data-capture-module.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst index 84bde0a5de4..4287241d46b 100644 --- a/doc/sphinx-guides/source/installation/data-capture-module.rst +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -1,10 +1,10 @@ Data Capture Module =================== -Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. Installation instructions can be found at https://github.com/sbgrid/data-capture-module +Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. Installation instructions can be found at https://github.com/sbgrid/data-capture-module . -Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting metioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate your DCM, so that Dataverse can download rsync scripts for your users. +Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting mentioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. -As of this writing, the only way to download an rsync script is via API using a URL such as the one below: +As of this writing, the only way to download an rsync script is via API using a URL like the one below: ``curl http://localhost:8080/api/datasets/{id}/dataCaptureModule/rsync`` From 4126ad1fcf191b56b8ba7c26fc637873ea783f63 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 30 May 2017 10:54:13 -0400 Subject: [PATCH 09/17] return more helpful message when DCM is broken #3725 --- .../harvard/iq/dataverse/api/Datasets.java | 7 ++++++- .../harvard/iq/dataverse/util/EjbUtil.java | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9bf78370cdc..342de0d52c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -55,6 +55,7 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -67,6 +68,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; +import javax.ejb.EJBException; import javax.inject.Inject; import javax.json.Json; import javax.json.JsonArrayBuilder; @@ -596,12 +598,15 @@ public Response getRsync(@PathParam("identifier") String id) { if (!DataCaptureModuleUtil.rsyncSupportEnabled(settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { return error(Response.Status.METHOD_NOT_ALLOWED, SettingsServiceBean.Key.UploadMethods + " does not contain " + SystemConfig.FileUploadMethods.RSYNC + "."); } + Dataset dataset = null; try { - Dataset dataset = findDatasetOrDie(id); + dataset = findDatasetOrDie(id); ScriptRequestResponse scriptRequestResponse = execCommand(new RequestRsyncScriptCommand(createDataverseRequest(findUserOrDie()), dataset)); return ok(scriptRequestResponse.getScript(), MediaType.valueOf(MediaType.TEXT_PLAIN)); } catch (WrappedResponse wr) { return wr.getResponse(); + } catch (EJBException ex) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "Something went wrong attempting to download rsync script: " + EjbUtil.ejbExceptionToString(ex)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java new file mode 100644 index 00000000000..cf337b0a020 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java @@ -0,0 +1,19 @@ +package edu.harvard.iq.dataverse.util; + +import javax.ejb.EJBException; + +public class EjbUtil { + + /** + * @param ex An EJBException. + * @return The message from the root cause of the EJBException as a String. + */ + public static String ejbExceptionToString(EJBException ex) { + Throwable cause = ex; + // An EJBException always has a cause. It won't be null. + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause.getLocalizedMessage(); + } +} From 85952cbb7c0e6a28a5fad8592031e279f072eaf9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 30 May 2017 15:20:22 -0400 Subject: [PATCH 10/17] better automated tests and error handling #3725 --- .../DataCaptureModuleException.java | 9 ++ .../DataCaptureModuleServiceBean.java | 25 ++--- .../DataCaptureModuleUtil.java | 17 +++- .../command/impl/CreateDatasetCommand.java | 2 + .../impl/RequestRsyncScriptCommand.java | 31 ++++-- .../DataCaptureModuleServiceBeanIT.java | 41 ++------ .../DataCaptureModuleUtilTest.java | 12 +++ .../impl/RequestRsyncScriptCommandTest.java | 98 +++++++++++++++++++ 8 files changed, 182 insertions(+), 53 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java new file mode 100644 index 00000000000..3329d92b7a9 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java @@ -0,0 +1,9 @@ +package edu.harvard.iq.dataverse.datacapturemodule; + +public class DataCaptureModuleException extends Exception { + + public DataCaptureModuleException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java index c28bb96e9d1..a2753464594 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java @@ -7,10 +7,8 @@ import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DataCaptureModuleUrl; import java.io.Serializable; -import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; @@ -30,19 +28,15 @@ public class DataCaptureModuleServiceBean implements Serializable { */ public static long millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls = 500; - @EJB - SettingsServiceBean settingsService; - // TODO: Do we care about authenticating to the DCM? If not, no need for AuthenticatedUser here. - public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset) { - String dcmBaseUrl = settingsService.getValueForKey(DataCaptureModuleUrl); + public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { if (dcmBaseUrl == null) { throw new RuntimeException("Problem POSTing JSON to Data Capture Module. The '" + DataCaptureModuleUrl + "' setting has not been configured."); } return makeUploadRequest(dcmBaseUrl, user, dataset); } - public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, AuthenticatedUser user, Dataset dataset) { + public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, AuthenticatedUser user, Dataset dataset) throws DataCaptureModuleException { String uploadRequestUrl = dcmBaseUrl + "/ur.py"; String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(user, dataset).toString(); // curl -H 'Content-Type: application/json' -X POST -d '{"datasetId":"42", "userId":"1642","datasetIdentifier":"42"}' http://localhost/ur.py @@ -54,13 +48,13 @@ public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, Authent UploadRequestResponse uploadRequestResponse = DataCaptureModuleUtil.makeUploadRequest(uploadRequest); return uploadRequestResponse; } catch (UnirestException ex) { - logger.info("Error calling " + uploadRequestUrl + ": " + ex); - return null; + String error = "Error calling " + uploadRequestUrl + ": " + ex; + logger.info(error); + throw new DataCaptureModuleException(error, ex); } } - public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset) { - String dcmBaseUrl = settingsService.getValueForKey(DataCaptureModuleUrl); + public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { if (dcmBaseUrl == null) { throw new RuntimeException("Problem GETing JSON to Data Capture Module for dataset " + dataset.getId() + " The '" + DataCaptureModuleUrl + "' setting has not been configured."); } @@ -68,7 +62,7 @@ public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset) { } - public static ScriptRequestResponse getRsyncScriptForDataset(String dcmBaseUrl, long datasetId) { + public static ScriptRequestResponse getRsyncScriptForDataset(String dcmBaseUrl, long datasetId) throws DataCaptureModuleException { String scriptRequestUrl = dcmBaseUrl + "/sr.py"; try { HttpResponse scriptRequest = Unirest.post(scriptRequestUrl) @@ -76,8 +70,9 @@ public static ScriptRequestResponse getRsyncScriptForDataset(String dcmBaseUrl, .asJson(); return DataCaptureModuleUtil.getScriptFromRequest(scriptRequest); } catch (UnirestException ex) { - logger.info("Error calling " + scriptRequestUrl + ": " + ex); - return null; + String error = "Error calling " + scriptRequestUrl + ": " + ex; + logger.info(error); + throw new DataCaptureModuleException(error, ex); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index b3ad9d3b1cb..000db761be7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -15,7 +15,7 @@ public class DataCaptureModuleUtil { private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { -// if (uploadMethodsSettings != null && uploadMethodsSettings.contains(SystemConfig.FileUploadMethods.RSYNC.toString())) { + logger.fine("uploadMethodsSettings: " + uploadMethodsSettings); if (uploadMethodsSettings != null && SystemConfig.FileUploadMethods.RSYNC.toString().equals(uploadMethodsSettings)) { return true; } else { @@ -54,4 +54,19 @@ static UploadRequestResponse makeUploadRequest(HttpResponse uploadReques return new UploadRequestResponse(uploadRequest.getStatus(), body); } + public static String getMessageFromException(DataCaptureModuleException ex) { + if (ex == null) { + return "DataCaptureModuleException was null!"; + } + Throwable cause = ex.getCause(); + if (cause == null) { + return ex.toString(); + } + String message = ex.getMessage(); + if (message == null) { + return cause.toString(); + } + return message + " was caused by " + cause.getMessage(); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java index eae995893d9..13bd1b11738 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetCommand.java @@ -196,6 +196,7 @@ by the Dataset page (in CREATE mode), it already has the persistent ctxt.templates().incrementUsageCount(template.getId()); } + logger.fine("Checking if rsync support is enabled."); if (DataCaptureModuleUtil.rsyncSupportEnabled(ctxt.settings().getValueForKey(SettingsServiceBean.Key.UploadMethods))) { try { ScriptRequestResponse scriptRequestResponse = ctxt.engine().submit(new RequestRsyncScriptCommand(getRequest(), savedDataset)); @@ -204,6 +205,7 @@ by the Dataset page (in CREATE mode), it already has the persistent logger.info("Problem getting rsync script: " + ex.getLocalizedMessage()); } } + logger.fine("Done with rsync request, if any."); try { /** diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index c7f02b68708..c277fa7ca76 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -7,12 +7,15 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleException; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.datacapturemodule.UploadRequestResponse; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DataCaptureModuleUrl; import java.util.Collections; import java.util.logging.Logger; @@ -39,6 +42,13 @@ public RequestRsyncScriptCommand(DataverseRequest requestArg, Dataset datasetArg @Override public ScriptRequestResponse execute(CommandContext ctxt) throws CommandException { + if (request == null) { + throw new PermissionException("DataverseRequest cannot be null.", this, Collections.singleton(Permission.AddDataset), dataset); + } + String dcmBaseUrl = ctxt.settings().getValueForKey(DataCaptureModuleUrl); + if (dcmBaseUrl == null) { + throw new RuntimeException(DataCaptureModuleUrl + " is null!"); + } User user = request.getUser(); if (!(user instanceof AuthenticatedUser)) { /** @@ -50,24 +60,33 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio } AuthenticatedUser au = (AuthenticatedUser) user; String errorPreamble = "User id " + au.getId() + " had a problem retrieving rsync script for dataset id " + dataset.getId() + " from Data Capture Module."; - UploadRequestResponse uploadRequestResponse = ctxt.dataCaptureModule().requestRsyncScriptCreation(au, dataset); + UploadRequestResponse uploadRequestResponse = null; + try { + uploadRequestResponse = ctxt.dataCaptureModule().requestRsyncScriptCreation(au, dataset, dcmBaseUrl); + } catch (DataCaptureModuleException ex) { + throw new RuntimeException("Problem making upload request to Data Capture Module: " + DataCaptureModuleUtil.getMessageFromException(ex)); + } int statusCode = uploadRequestResponse.getHttpStatusCode(); String response = uploadRequestResponse.getResponse(); if (statusCode != 200) { - throw new RuntimeException("When making the upload request, rather than 200 the status code was " + statusCode + ". The body was \'" + response + "\'. We cannont proceed. Returning."); + throw new RuntimeException("When making the upload request, rather than 200 the status code was " + statusCode + ". The body was \'" + response + "\'. We cannot proceed. Returning."); } long millisecondsToSleep = DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls; logger.fine("Message from Data Caputure Module upload request endpoint: " + response + ". Sleeping " + millisecondsToSleep + " milliseconds before making rsync script request."); try { Thread.sleep(millisecondsToSleep); } catch (InterruptedException ex) { - throw new RuntimeException(errorPreamble + "Unable to wait " + millisecondsToSleep + " milliseconds: " + ex.getLocalizedMessage()); + throw new RuntimeException(errorPreamble + " Unable to wait " + millisecondsToSleep + " milliseconds: " + ex.getLocalizedMessage()); + } + ScriptRequestResponse scriptRequestResponse = null; + try { + scriptRequestResponse = ctxt.dataCaptureModule().retreiveRequestedRsyncScript(dataset, dcmBaseUrl); + } catch (DataCaptureModuleException ex) { + throw new RuntimeException("Problem making script request to Data Capture Module: " + DataCaptureModuleUtil.getMessageFromException(ex)); } - - ScriptRequestResponse scriptRequestResponse = ctxt.dataCaptureModule().retreiveRequestedRsyncScript(dataset); String script = scriptRequestResponse.getScript(); if (script == null || script.isEmpty()) { - throw new RuntimeException(errorPreamble + "The script was null or empty."); + throw new RuntimeException(errorPreamble + " The script was null or empty."); } logger.fine("script for dataset " + dataset.getId() + ": " + script); return scriptRequestResponse; diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java index 1c87156b3d6..9109442e04d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java @@ -7,47 +7,26 @@ import java.util.Calendar; import java.util.TimeZone; import java.util.logging.Logger; -import junit.framework.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.junit.Test; +import static java.lang.Thread.sleep; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +/** + * These tests are not expected to pass unless you have a Data Capture Module + * (DCM) installed and configured properly. They are intended to help a + * developer get set up for DCM development. + */ public class DataCaptureModuleServiceBeanIT { private static final Logger logger = Logger.getLogger(DataCaptureModuleServiceBeanIT.class.getCanonicalName()); @Test - public void testUploadRequestWorking() { - AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); - Dataset dataset = new Dataset(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - long timeInMillis = calendar.getTimeInMillis(); - dataset.setId(timeInMillis); - UploadRequestResponse uploadRequestResponse = DataCaptureModuleServiceBean.makeUploadRequest("http://localhost:8888", user, dataset); - assertEquals(200, uploadRequestResponse.getHttpStatusCode()); - assertTrue(uploadRequestResponse.getResponse().contains("recieved")); - assertEquals("\nrecieved\n", uploadRequestResponse.getResponse()); - } - - @Test - public void testScriptRequestWorking() { - long expectedToWork = 3813; - ScriptRequestResponse scriptRequestResponseGood = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", expectedToWork); - System.out.println("script: " + scriptRequestResponseGood.getScript()); - Assert.assertTrue(scriptRequestResponseGood.getScript().startsWith("#!")); - } - - @Test - public void testScriptRequestNotWorking() { - long notExpectedToWork = Long.MAX_VALUE; - ScriptRequestResponse scriptRequestResponseBad = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", notExpectedToWork); - assertNull(scriptRequestResponseBad.getScript()); - } - - @Test - public void testBothSteps() throws InterruptedException { + public void testUploadRequestAndScriptRequest() throws InterruptedException, DataCaptureModuleException { // Step 1: Upload request AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); Dataset dataset = new Dataset(); diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java index 84526d1a864..7bf1e618ff2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java @@ -93,4 +93,16 @@ public void testMakeUploadRequest() throws UnsupportedEncodingException { assertEquals("received", result.getResponse()); } + @Test + public void testGetMessageFromException() { + System.out.println("getMessageFromException"); + // preferred form + assertEquals("message1 was caused by innerExceptionMessage", DataCaptureModuleUtil.getMessageFromException(new DataCaptureModuleException("message1", new NullPointerException("innerExceptionMessage")))); + // suboptimal messages + assertEquals("message1 was caused by null", DataCaptureModuleUtil.getMessageFromException(new DataCaptureModuleException("message1", new NullPointerException()))); + assertEquals("java.lang.NullPointerException", DataCaptureModuleUtil.getMessageFromException(new DataCaptureModuleException(null, new NullPointerException()))); + assertEquals("DataCaptureModuleException was null!", DataCaptureModuleUtil.getMessageFromException(null)); + assertEquals("edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleException", DataCaptureModuleUtil.getMessageFromException(new DataCaptureModuleException(null, null))); + assertEquals("edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleException: message1", DataCaptureModuleUtil.getMessageFromException(new DataCaptureModuleException("message1", null))); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java new file mode 100644 index 00000000000..706f5897169 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java @@ -0,0 +1,98 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; +import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; +import edu.harvard.iq.dataverse.datacapturemodule.UploadRequestResponse; +import edu.harvard.iq.dataverse.engine.TestCommandContext; +import edu.harvard.iq.dataverse.engine.TestDataverseEngine; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.mocks.MocksFactory; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DataCaptureModuleUrl; +import javax.servlet.http.HttpServletRequest; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +public class RequestRsyncScriptCommandTest { + + private TestDataverseEngine testEngine; + Dataset dataset; + + public RequestRsyncScriptCommandTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + testEngine = new TestDataverseEngine(new TestCommandContext() { + + @Override + public DataCaptureModuleServiceBean dataCaptureModule() { + return new DataCaptureModuleServiceBean() { + + @Override + public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset, String dcmBaseUrl) { + return new UploadRequestResponse(200, "myResponse"); + } + + @Override + public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset, String dcmBaseUrl) { + int httpStatusCode = 200; + long datasetId = dataset.getId(); + long userId = 123l; + String script = "theScript"; + ScriptRequestResponse scriptRequestResponse = new ScriptRequestResponse(httpStatusCode, datasetId, userId, script); + return scriptRequestResponse; + } + + }; + + } + + @Override + public SettingsServiceBean settings() { + return new MockSettingsSvc(); + } + }); + } + + @After + public void tearDown() { + } + + @Test + public void testHappyPath() throws Exception { + dataset = new Dataset(); + dataset.setId(42l); + HttpServletRequest aHttpServletRequest = null; + DataverseRequest dataverseRequest = new DataverseRequest(MocksFactory.makeAuthenticatedUser("First", "Last"), aHttpServletRequest); + ScriptRequestResponse scriptRequestResponse = testEngine.submit(new RequestRsyncScriptCommand(dataverseRequest, dataset)); + assertEquals("theScript", scriptRequestResponse.getScript()); + } + + private static class MockSettingsSvc extends SettingsServiceBean { + + @Override + public String getValueForKey(SettingsServiceBean.Key key) { + switch (key) { + case DataCaptureModuleUrl: + return "http://localhost:8888"; + default: + return null; + } + } + } +} From 967c12703a9ffd910e5def4a2d902ce06b337d55 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 30 May 2017 15:38:51 -0400 Subject: [PATCH 11/17] consolidate duplicate code #3725 --- .../DataCaptureModuleServiceBean.java | 19 ++----------------- .../impl/RequestRsyncScriptCommand.java | 1 + .../DataCaptureModuleServiceBeanIT.java | 6 ++++-- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java index a2753464594..361e75345f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java @@ -7,14 +7,13 @@ import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import edu.harvard.iq.dataverse.Dataset; -import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DataCaptureModuleUrl; import java.io.Serializable; import javax.ejb.Stateless; import javax.inject.Named; /** * This class contains all the methods that have external runtime dependencies - * such as the Data Capture Module itself and PostgreSQL. + * such as the Data Capture Module itself. */ @Stateless @Named @@ -30,13 +29,6 @@ public class DataCaptureModuleServiceBean implements Serializable { // TODO: Do we care about authenticating to the DCM? If not, no need for AuthenticatedUser here. public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { - if (dcmBaseUrl == null) { - throw new RuntimeException("Problem POSTing JSON to Data Capture Module. The '" + DataCaptureModuleUrl + "' setting has not been configured."); - } - return makeUploadRequest(dcmBaseUrl, user, dataset); - } - - public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, AuthenticatedUser user, Dataset dataset) throws DataCaptureModuleException { String uploadRequestUrl = dcmBaseUrl + "/ur.py"; String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(user, dataset).toString(); // curl -H 'Content-Type: application/json' -X POST -d '{"datasetId":"42", "userId":"1642","datasetIdentifier":"42"}' http://localhost/ur.py @@ -55,14 +47,7 @@ public static UploadRequestResponse makeUploadRequest(String dcmBaseUrl, Authent } public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { - if (dcmBaseUrl == null) { - throw new RuntimeException("Problem GETing JSON to Data Capture Module for dataset " + dataset.getId() + " The '" + DataCaptureModuleUrl + "' setting has not been configured."); - } - return getRsyncScriptForDataset(dcmBaseUrl, dataset.getId()); - - } - - public static ScriptRequestResponse getRsyncScriptForDataset(String dcmBaseUrl, long datasetId) throws DataCaptureModuleException { + Long datasetId = dataset.getId(); String scriptRequestUrl = dcmBaseUrl + "/sr.py"; try { HttpResponse scriptRequest = Unirest.post(scriptRequestUrl) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index c277fa7ca76..8fb43775531 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -58,6 +58,7 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio throw new PermissionException("This command can only be called by an AuthenticatedUser, not " + user, this, Collections.singleton(Permission.AddDataset), dataset); } + // We need an AuthenticatedUser so we can pass its database id to the DCM. AuthenticatedUser au = (AuthenticatedUser) user; String errorPreamble = "User id " + au.getId() + " had a problem retrieving rsync script for dataset id " + dataset.getId() + " from Data Capture Module."; UploadRequestResponse uploadRequestResponse = null; diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java index 9109442e04d..d5519172bc9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java @@ -27,13 +27,15 @@ public class DataCaptureModuleServiceBeanIT { @Test public void testUploadRequestAndScriptRequest() throws InterruptedException, DataCaptureModuleException { + DataCaptureModuleServiceBean dataCaptureModuleServiceBean = new DataCaptureModuleServiceBean(); + // Step 1: Upload request AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); Dataset dataset = new Dataset(); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); long timeInMillis = calendar.getTimeInMillis(); dataset.setId(timeInMillis); - UploadRequestResponse uploadRequestResponse = DataCaptureModuleServiceBean.makeUploadRequest("http://localhost:8888", user, dataset); + UploadRequestResponse uploadRequestResponse = dataCaptureModuleServiceBean.requestRsyncScriptCreation(user, dataset, "http://localhost:8888"); assertEquals(200, uploadRequestResponse.getHttpStatusCode()); assertTrue(uploadRequestResponse.getResponse().contains("recieved")); assertEquals("\nrecieved\n", uploadRequestResponse.getResponse()); @@ -42,7 +44,7 @@ public void testUploadRequestAndScriptRequest() throws InterruptedException, Dat sleep(DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls); // Step 2: Script request. - ScriptRequestResponse scriptRequestResponseGood = DataCaptureModuleServiceBean.getRsyncScriptForDataset("http://localhost:8888", dataset.getId()); + ScriptRequestResponse scriptRequestResponseGood = dataCaptureModuleServiceBean.retreiveRequestedRsyncScript(dataset, "http://localhost:8888"); System.out.println("script: " + scriptRequestResponseGood.getScript()); assertNotNull(scriptRequestResponseGood.getScript()); assertTrue(scriptRequestResponseGood.getScript().startsWith("#!")); From 087f87608af3eb2202b078f0ba16d0c366754bf6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 30 May 2017 16:09:50 -0400 Subject: [PATCH 12/17] consolidate upload and script request path definitions #3725 --- .../DataCaptureModuleServiceBean.java | 16 ++++++---------- .../datacapturemodule/DataCaptureModuleUtil.java | 2 +- .../command/impl/RequestRsyncScriptCommand.java | 9 +++++---- .../DataCaptureModuleServiceBeanIT.java | 9 ++++++--- .../impl/RequestRsyncScriptCommandTest.java | 6 ++---- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java index 361e75345f4..a4041f7fc57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBean.java @@ -1,12 +1,10 @@ package edu.harvard.iq.dataverse.datacapturemodule; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import java.util.logging.Logger; import com.mashape.unirest.http.HttpResponse; import com.mashape.unirest.http.JsonNode; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; -import edu.harvard.iq.dataverse.Dataset; import java.io.Serializable; import javax.ejb.Stateless; import javax.inject.Named; @@ -26,13 +24,12 @@ public class DataCaptureModuleServiceBean implements Serializable { * making this configurable. */ public static long millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls = 500; + public static String uploadRequestPath = "/ur.py"; + public static String scriptRequestPath = "/sr.py"; // TODO: Do we care about authenticating to the DCM? If not, no need for AuthenticatedUser here. - public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { - String uploadRequestUrl = dcmBaseUrl + "/ur.py"; - String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(user, dataset).toString(); - // curl -H 'Content-Type: application/json' -X POST -d '{"datasetId":"42", "userId":"1642","datasetIdentifier":"42"}' http://localhost/ur.py - logger.info("JSON to send to Data Capture Module: " + jsonString); + public UploadRequestResponse requestRsyncScriptCreation(String jsonString, String uploadRequestUrl) throws DataCaptureModuleException { + logger.fine("requestRsyncScriptCreation using JSON string: " + jsonString + " and sending to " + uploadRequestUrl); try { HttpResponse uploadRequest = Unirest.post(uploadRequestUrl) .body(jsonString) @@ -46,9 +43,8 @@ public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, } } - public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset, String dcmBaseUrl) throws DataCaptureModuleException { - Long datasetId = dataset.getId(); - String scriptRequestUrl = dcmBaseUrl + "/sr.py"; + public ScriptRequestResponse retreiveRequestedRsyncScript(long datasetId, String scriptRequestUrl) throws DataCaptureModuleException { + logger.fine("retreiveRequestedRsyncScript using dataset id + " + datasetId + " to " + scriptRequestUrl); try { HttpResponse scriptRequest = Unirest.post(scriptRequestUrl) .field("datasetIdentifier", datasetId) diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index 000db761be7..7aeffc5870b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -23,7 +23,7 @@ public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { } } - static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { + public static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { JsonObjectBuilder jab = Json.createObjectBuilder(); // // The general rule should be to always pass the user id and dataset id to the DCM. jab.add("userId", user.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index 8fb43775531..1bfb030188b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -59,11 +59,12 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio this, Collections.singleton(Permission.AddDataset), dataset); } // We need an AuthenticatedUser so we can pass its database id to the DCM. - AuthenticatedUser au = (AuthenticatedUser) user; - String errorPreamble = "User id " + au.getId() + " had a problem retrieving rsync script for dataset id " + dataset.getId() + " from Data Capture Module."; + AuthenticatedUser authenticatedUser = (AuthenticatedUser) user; + String errorPreamble = "User id " + authenticatedUser.getId() + " had a problem retrieving rsync script for dataset id " + dataset.getId() + " from Data Capture Module."; + String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(authenticatedUser, dataset).toString(); UploadRequestResponse uploadRequestResponse = null; try { - uploadRequestResponse = ctxt.dataCaptureModule().requestRsyncScriptCreation(au, dataset, dcmBaseUrl); + uploadRequestResponse = ctxt.dataCaptureModule().requestRsyncScriptCreation(jsonString, dcmBaseUrl + DataCaptureModuleServiceBean.uploadRequestPath); } catch (DataCaptureModuleException ex) { throw new RuntimeException("Problem making upload request to Data Capture Module: " + DataCaptureModuleUtil.getMessageFromException(ex)); } @@ -81,7 +82,7 @@ public ScriptRequestResponse execute(CommandContext ctxt) throws CommandExceptio } ScriptRequestResponse scriptRequestResponse = null; try { - scriptRequestResponse = ctxt.dataCaptureModule().retreiveRequestedRsyncScript(dataset, dcmBaseUrl); + scriptRequestResponse = ctxt.dataCaptureModule().retreiveRequestedRsyncScript(dataset.getId(), dcmBaseUrl + DataCaptureModuleServiceBean.scriptRequestPath); } catch (DataCaptureModuleException ex) { throw new RuntimeException("Problem making script request to Data Capture Module: " + DataCaptureModuleUtil.getMessageFromException(ex)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java index d5519172bc9..742aecb508a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleServiceBeanIT.java @@ -27,15 +27,18 @@ public class DataCaptureModuleServiceBeanIT { @Test public void testUploadRequestAndScriptRequest() throws InterruptedException, DataCaptureModuleException { + String dcmBaseUrl = "http://localhost:8888"; DataCaptureModuleServiceBean dataCaptureModuleServiceBean = new DataCaptureModuleServiceBean(); // Step 1: Upload request - AuthenticatedUser user = makeAuthenticatedUser("Lauren", "Ipsum"); + AuthenticatedUser authenticatedUser = makeAuthenticatedUser("Lauren", "Ipsum"); Dataset dataset = new Dataset(); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); long timeInMillis = calendar.getTimeInMillis(); dataset.setId(timeInMillis); - UploadRequestResponse uploadRequestResponse = dataCaptureModuleServiceBean.requestRsyncScriptCreation(user, dataset, "http://localhost:8888"); + String jsonString = DataCaptureModuleUtil.generateJsonForUploadRequest(authenticatedUser, dataset).toString(); + logger.info("jsonString: " + jsonString); + UploadRequestResponse uploadRequestResponse = dataCaptureModuleServiceBean.requestRsyncScriptCreation(jsonString, dcmBaseUrl + DataCaptureModuleServiceBean.uploadRequestPath); assertEquals(200, uploadRequestResponse.getHttpStatusCode()); assertTrue(uploadRequestResponse.getResponse().contains("recieved")); assertEquals("\nrecieved\n", uploadRequestResponse.getResponse()); @@ -44,7 +47,7 @@ public void testUploadRequestAndScriptRequest() throws InterruptedException, Dat sleep(DataCaptureModuleServiceBean.millisecondsToSleepBetweenUploadRequestAndScriptRequestCalls); // Step 2: Script request. - ScriptRequestResponse scriptRequestResponseGood = dataCaptureModuleServiceBean.retreiveRequestedRsyncScript(dataset, "http://localhost:8888"); + ScriptRequestResponse scriptRequestResponseGood = dataCaptureModuleServiceBean.retreiveRequestedRsyncScript(dataset.getId(), dcmBaseUrl + DataCaptureModuleServiceBean.scriptRequestPath); System.out.println("script: " + scriptRequestResponseGood.getScript()); assertNotNull(scriptRequestResponseGood.getScript()); assertTrue(scriptRequestResponseGood.getScript().startsWith("#!")); diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java index 706f5897169..d7b1a131d84 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommandTest.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.datacapturemodule.UploadRequestResponse; @@ -44,14 +43,13 @@ public DataCaptureModuleServiceBean dataCaptureModule() { return new DataCaptureModuleServiceBean() { @Override - public UploadRequestResponse requestRsyncScriptCreation(AuthenticatedUser user, Dataset dataset, String dcmBaseUrl) { + public UploadRequestResponse requestRsyncScriptCreation(String jsonString, String dcmBaseUrl) { return new UploadRequestResponse(200, "myResponse"); } @Override - public ScriptRequestResponse retreiveRequestedRsyncScript(Dataset dataset, String dcmBaseUrl) { + public ScriptRequestResponse retreiveRequestedRsyncScript(long datasetId, String dcmBaseUrl) { int httpStatusCode = 200; - long datasetId = dataset.getId(); long userId = 123l; String script = "theScript"; ScriptRequestResponse scriptRequestResponse = new ScriptRequestResponse(httpStatusCode, datasetId, userId, script); From 426cf45a4b98b1e4cadb0a3e941f96aa158a185e Mon Sep 17 00:00:00 2001 From: Susan Sons Date: Sun, 4 Jun 2017 14:20:01 -0400 Subject: [PATCH 13/17] Updated CONTRIBUTING.md in an attempt to appeal to inexperienced contributors. --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba3532d3b6f..51081f80275 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,8 @@ # Contributing to Dataverse -Thank you for your interest in contributing to Dataverse! We welcome contributions of ideas, bug reports, usability testing, documentation, code, and more! +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone, you don't need permission to participate, just jump in using the resources below. If you have questions, reach out to us on the [#dataverse IRC channel][], and hang around a while as it may take time for community members to de-idle. + +We aren't just looking for developers, there are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability testing, documentation, code, and more! ## Ideas/Feature Requests From bd3cd09d87f4b5610d8212cd2709491a67c8199d Mon Sep 17 00:00:00 2001 From: Derek Murphy Date: Mon, 5 Jun 2017 13:56:52 -0400 Subject: [PATCH 14/17] Small edits to phrasing [ref: #3882] --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 51081f80275..f7b83f8c492 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,8 @@ # Contributing to Dataverse -Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone, you don't need permission to participate, just jump in using the resources below. If you have questions, reach out to us on the [#dataverse IRC channel][], and hang around a while as it may take time for community members to de-idle. +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. You don't need permission to participate, just jump in using the resources below. If you have questions, reach out to us on the [#dataverse IRC channel][], and hang around a while, as it may take time for community members to de-idle. -We aren't just looking for developers, there are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability testing, documentation, code, and more! +We aren't just looking for developers, there are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! ## Ideas/Feature Requests From 89f21ac86af0f61ef69d0084936361ec425e10d9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 6 Jun 2017 10:01:59 -0400 Subject: [PATCH 15/17] document nicer DOI form of API endpoint #3725 --- .../source/installation/data-capture-module.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst index 4287241d46b..4aba3c6c32b 100644 --- a/doc/sphinx-guides/source/installation/data-capture-module.rst +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -3,8 +3,13 @@ Data Capture Module Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. Installation instructions can be found at https://github.com/sbgrid/data-capture-module . -Once you have installed a DCM, you will need to configure Dataverse with its URL using the ``:DataCaptureModuleUrl`` setting mentioned on the :doc:`config` section and set the ``:UploadMethods`` setting to ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. +Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`config` section: -As of this writing, the only way to download an rsync script is via API using a URL like the one below: +- ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. +- ``:UploadMethods`` should be set to ``dcm/rsync+ssh``. + +This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. -``curl http://localhost:8080/api/datasets/{id}/dataCaptureModule/rsync`` +The rsync script can be downloaded via API using an authorized API token. In the curl example below, substitute ``{persistentId}`` with a DOI or Handle: + +``curl -H "X-Dataverse-key: $API_TOKEN" http://localhost:8080/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId={persistentId}`` From de03d5e74b6a56e87b82d5856b0a30acfe562f13 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 6 Jun 2017 11:09:16 -0400 Subject: [PATCH 16/17] require edit permission on dataset (more strict) #3725 --- .../engine/command/impl/RequestRsyncScriptCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index 1bfb030188b..2f590b91dc0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -26,7 +26,7 @@ * "actiontype" in the actionlogrecord rather than "InternalError" if you throw * a CommandExecutionException. */ -@RequiredPermissions(Permission.AddDataset) +@RequiredPermissions(Permission.EditDataset) public class RequestRsyncScriptCommand extends AbstractCommand { private static final Logger logger = Logger.getLogger(RequestRsyncScriptCommand.class.getCanonicalName()); From 44572fd5a019a11ab7736a42a47a61448858113f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 6 Jun 2017 15:04:02 -0400 Subject: [PATCH 17/17] clarify that the rsync script is downloaded from Dataverse #3725 --- doc/sphinx-guides/source/installation/data-capture-module.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst index 4aba3c6c32b..38039fdaf0f 100644 --- a/doc/sphinx-guides/source/installation/data-capture-module.rst +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -10,6 +10,6 @@ Once you have installed a DCM, you will need to configure two database settings This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. -The rsync script can be downloaded via API using an authorized API token. In the curl example below, substitute ``{persistentId}`` with a DOI or Handle: +The rsync script can be downloaded from Dataverse via API using an authorized API token. In the curl example below, substitute ``{persistentId}`` with a DOI or Handle: -``curl -H "X-Dataverse-key: $API_TOKEN" http://localhost:8080/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId={persistentId}`` +``curl -H "X-Dataverse-key: $API_TOKEN" https://dataverse.example.edu/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId={persistentId}``