Merge branch 'develop' into 8932-ct-base-image

doc/release-notes/8715-importddi-termofuse.md

@@ -0,0 +1 @@
+Terms of Use is now imported when using DDI format through harvesting or the native API. (Issue #8715, PR #8743)

doc/release-notes/8882-shib-affiliation.md

@@ -0,0 +1,4 @@
+## New DB Settings
+The following DB settings have been added:
+- `:ShibAffiliationOrder`  - Select the first or last entry in an Affiliation array
+- `:ShibAffiliationSeparator` (default: ";") - Set the separator for the Affiliation array

doc/sphinx-guides/source/_static/api/ddi_dataset.xml

@@ -142,6 +142,7 @@
     </method>
     <dataAccs>
       <notes type="DVN:TOA" level="dv">Terms of Access</notes>
+      <notes type="DVN:TOU" level="dv">Terms of Use</notes>
       <setAvail>
         <accsPlac>Data Access Place</accsPlac>
         <origArch>Original Archive</origArch>

doc/sphinx-guides/source/installation/config.rst

@@ -2456,6 +2456,30 @@ In the case you get garbled characters in Shibboleth-supplied fields (e.g. given
 
 If you managed to get correct accented characters from shibboleth while this setting is _false_, please contact us with your application server and Shibboleth configuration!
 
+:ShibAffiliationOrder
++++++++++++++++++++++
+
+Will select the last or first value of an array in affiliation, the array separator can be set using ``:ShibAffiliationSeparator`` .
+
+To select the last value : 
+
+``curl -X PUT -d "lastAffiliation" http://localhost:8080/api/admin/settings/:ShibAffiliationOrder``
+
+To select the first value :
+
+``curl -X PUT -d "firstAffiliation" http://localhost:8080/api/admin/settings/:ShibAffiliationOrder``
+
+
+:ShibAffiliationSeparator
++++++++++++++++++++++++++
+
+Set the separator to be used for ``:ShibAffiliationOrder``.
+Default separator : ";"
+
+To change the separator :
+
+``curl -X PUT -d ";" http://localhost:8080/api/admin/settings/:ShibAffiliationSeparator``
+
 .. _:ComputeBaseUrl:
 
 :ComputeBaseUrl

doc/sphinx-guides/source/installation/prep.rst

@@ -27,6 +27,7 @@ Advanced Installation
 There are some community-lead projects to use configuration management tools such as Ansible and Puppet to automate the installation and configuration of the Dataverse Software, but support for these solutions is limited to what the Dataverse Community can offer as described in each project's webpage:
 
 - https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible
+- https://gitlab.com/lip-computing/dataverse
 - https://github.com/IQSS/dataverse-puppet
 
 (Please note that the "dataverse-ansible" repo is used in a script that allows the Dataverse Software to be installed on Amazon Web Services (AWS) from arbitrary GitHub branches as described in the :doc:`/developers/deployment` section of the Developer Guide.)

src/main/java/edu/harvard/iq/dataverse/Shib.java

@@ -218,7 +218,26 @@ public void init() {
             ? getValueFromAssertion(shibAffiliationAttribute)
             : shibService.getAffiliation(shibIdp, shibService.getDevShibAccountType());
 
+
         if (affiliation != null) {
+            String ShibAffiliationSeparator = settingsService.getValueForKey(SettingsServiceBean.Key.ShibAffiliationSeparator);
+            if (ShibAffiliationSeparator == null) {
+                ShibAffiliationSeparator = ";";
+            }
+            String ShibAffiliationOrder = settingsService.getValueForKey(SettingsServiceBean.Key.ShibAffiliationOrder);
+            if (ShibAffiliationOrder != null) {
+                if (ShibAffiliationOrder.equals("lastAffiliation")) {
+                    affiliation = affiliation.substring(affiliation.lastIndexOf(ShibAffiliationSeparator) + 1); //patch for affiliation array returning last part
+                }
+                else if (ShibAffiliationOrder.equals("firstAffiliation")) {
+                    try{
+                        affiliation = affiliation.substring(0,affiliation.indexOf(ShibAffiliationSeparator)); //patch for affiliation array returning first part
+                    }
+                    catch (Exception e){
+                        logger.info("Affiliation does not contain \"" + ShibAffiliationSeparator + "\"");
+                    }
+                }
+            }
             affiliationToDisplayAtConfirmation = affiliation;
             friendlyNameForInstitution = affiliation;
         }

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -82,6 +82,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.EjbUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
+import edu.harvard.iq.dataverse.util.MarkupChecker;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.bagit.OREMap;
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
@@ -3323,17 +3324,20 @@ public Response getDatasetVersionArchivalStatus(@PathParam("id") String datasetI
     @Consumes(MediaType.APPLICATION_JSON)
     @Path("/{id}/{version}/archivalStatus")
     public Response setDatasetVersionArchivalStatus(@PathParam("id") String datasetId,
-            @PathParam("version") String versionNumber, JsonObject update, @Context UriInfo uriInfo,
+            @PathParam("version") String versionNumber, String newStatus, @Context UriInfo uriInfo,
             @Context HttpHeaders headers) {
 
-        logger.fine(JsonUtil.prettyPrint(update));
+        logger.fine(newStatus);
         try {
             AuthenticatedUser au = findAuthenticatedUserOrDie();
 
             if (!au.isSuperuser()) {
                 return error(Response.Status.FORBIDDEN, "Superusers only.");
             }
-
+
+            //Verify we have valid json after removing any HTML tags (the status gets displayed in the UI, so we want plain text).
+            JsonObject update= JsonUtil.getJsonObject(MarkupChecker.stripAllTags(newStatus));
+
             if (update.containsKey(DatasetVersion.ARCHIVAL_STATUS) && update.containsKey(DatasetVersion.ARCHIVAL_STATUS_MESSAGE)) {
                 String status = update.getString(DatasetVersion.ARCHIVAL_STATUS);
                 if (status.equals(DatasetVersion.ARCHIVAL_STATUS_PENDING) || status.equals(DatasetVersion.ARCHIVAL_STATUS_FAILURE)
@@ -3364,8 +3368,9 @@ public Response setDatasetVersionArchivalStatus(@PathParam("id") String datasetI
             }
         } catch (WrappedResponse wr) {
             return wr.getResponse();
+        } catch (JsonException| IllegalStateException ex) {
+            return error(Status.BAD_REQUEST, "Unable to parse provided JSON");
         }
-
         return error(Status.BAD_REQUEST, "Unacceptable status format");
     }
 

src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java

@@ -1181,7 +1181,7 @@ private void processDataAccs(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro
                     String noteType = xmlr.getAttributeValue(null, "type");
                     if (NOTE_TYPE_TERMS_OF_USE.equalsIgnoreCase(noteType) ) {
                         if ( LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) {
-                            parseText(xmlr, "notes");
+                            dvDTO.setTermsOfUse(parseText(xmlr, "notes"));
                         }
                     } else  if (NOTE_TYPE_TERMS_OF_ACCESS.equalsIgnoreCase(noteType) ) {
                         if (LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) {

src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibServiceBean.java

@@ -64,6 +64,8 @@ public enum DevShibAccountType {
         UID_WITH_LEADING_SPACE,
         IDENTIFIER_WITH_LEADING_SPACE,
         MISSING_REQUIRED_ATTR,
+        ONE_AFFILIATION,
+        TWO_AFFILIATIONS,
     };
 
     public DevShibAccountType getDevShibAccountType() {
@@ -146,6 +148,14 @@ public void possiblyMutateRequestInDev(HttpServletRequest request) {
                 ShibUtil.mutateRequestForDevConstantMissingRequiredAttributes(request);
                 break;
 
+            case ONE_AFFILIATION:
+                ShibUtil.mutateRequestForDevConstantOneAffiliation(request);
+                break;
+
+            case TWO_AFFILIATIONS:
+                ShibUtil.mutateRequestForDevConstantTwoAffiliations(request);
+                break;
+
             default:
                 logger.info("Should never reach here");
                 break;

src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java

@@ -261,6 +261,28 @@ static void mutateRequestForDevConstantMissingRequiredAttributes(HttpServletRequ
         request.setAttribute(ShibUtil.usernameAttribute, "missing");
     }
 
+    static void mutateRequestForDevConstantOneAffiliation(HttpServletRequest request) {
+        request.setAttribute(ShibUtil.shibIdpAttribute, "https://fake.example.com/idp/shibboleth");
+        request.setAttribute(ShibUtil.uniquePersistentIdentifier, "oneAffiliation");
+        request.setAttribute(ShibUtil.firstNameAttribute, "Lurneen");
+        request.setAttribute(ShibUtil.lastNameAttribute, "Lumpkin");
+        request.setAttribute(ShibUtil.emailAttribute, "oneAffiliaton@mailinator.com");
+        request.setAttribute(ShibUtil.usernameAttribute, "oneAffiliaton");
+        // Affiliation. "ou" is the suggested attribute in :ShibAffiliationAttribute.
+        request.setAttribute("ou", "Beer-N-Brawl");
+    }
+
+    static void mutateRequestForDevConstantTwoAffiliations(HttpServletRequest request) {
+        request.setAttribute(ShibUtil.shibIdpAttribute, "https://fake.example.com/idp/shibboleth");
+        request.setAttribute(ShibUtil.uniquePersistentIdentifier, "twoAffiliatons");
+        request.setAttribute(ShibUtil.firstNameAttribute, "Lenny");
+        request.setAttribute(ShibUtil.lastNameAttribute, "Leonard");
+        request.setAttribute(ShibUtil.emailAttribute, "twoAffiliatons@mailinator.com");
+        request.setAttribute(ShibUtil.usernameAttribute, "twoAffiliatons");
+        // Affiliation. "ou" is the suggested attribute in :ShibAffiliationAttribute.
+        request.setAttribute("ou", "SNPP;Stonecutters");
+    }
+
     public static Map<String, String> getRandomUserStatic() {
         Map<String, String> fakeUser = new HashMap<>();
         String shortRandomString = UUID.randomUUID().toString().substring(0, 8);

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DRSSubmitToArchiveCommand.java

@@ -61,7 +61,7 @@
 public class DRSSubmitToArchiveCommand extends S3SubmitToArchiveCommand implements Command<DatasetVersion> {
 
     private static final Logger logger = Logger.getLogger(DRSSubmitToArchiveCommand.class.getName());
-    private static final String DRS_CONFIG = ":DRSArchivalConfig";
+    private static final String DRS_CONFIG = ":DRSArchiverConfig";
     private static final String ADMIN_METADATA = "admin_metadata";
     private static final String S3_BUCKET_NAME = "s3_bucket_name";
     private static final String S3_PATH = "s3_path";

src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java

@@ -425,6 +425,14 @@ Whether Harvesting (OAI) service is enabled
          * Convert shibboleth AJP attributes from ISO-8859-1 to UTF-8
          */
         ShibAttributeCharacterSetConversionEnabled,
+        /**
+         *Return the last or first value of an array of affiliation names
+         */
+        ShibAffiliationOrder,
+         /**
+         *Split the affiliation array on given string, default ";"
+         */
+        ShibAffiliationSeparator,
         /**
          * Validate physical files for all the datafiles in the dataset when publishing
          */

src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java

@@ -7,7 +7,7 @@
 
 import org.apache.commons.text.StringEscapeUtils;
 import org.jsoup.Jsoup;
-import org.jsoup.safety.Whitelist;
+import org.jsoup.safety.Safelist;
 import org.jsoup.parser.Parser;
 
 /**
@@ -20,8 +20,8 @@ public class MarkupChecker {
 
 
     /**
-     * Wrapper around Jsoup clean method with the basic White list
-     *   http://jsoup.org/cookbook/cleaning-html/whitelist-sanitizer
+     * Wrapper around Jsoup clean method with the basic Safe list
+     *   http://jsoup.org/cookbook/cleaning-html/safelist-sanitizer
      * @param unsafe
      * @return 
      */
@@ -33,18 +33,18 @@ public static String sanitizeBasicHTML(String unsafe) {
         // basic includes: a, b, blockquote, br, cite, code, dd, dl, dt, em, i, li, ol, p, pre, q, small, span, strike, strong, sub, sup, u, ul
         //Whitelist wl = Whitelist.basic().addTags("img", "h1", "h2", "h3", "kbd", "hr", "s", "del");  
 
-        Whitelist wl = Whitelist.basicWithImages().addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area").addAttributes("img", "usemap")
+        Safelist sl = Safelist.basicWithImages().addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area").addAttributes("img", "usemap")
                 .addAttributes("map", "name").addAttributes("area", "shape", "coords", "href", "title", "alt")
                 .addEnforcedAttribute("a", "target", "_blank");
 
-        return Jsoup.clean(unsafe, wl);
+        return Jsoup.clean(unsafe, sl);
 
     }
 
     /**
      * Strip all HTMl tags
      * 
-     * http://jsoup.org/apidocs/org/jsoup/safety/Whitelist.html#none%28%29
+     * http://jsoup.org/apidocs/org/jsoup/safety/Safelist.html#none
      * 
      * @param unsafe
      * @return 
@@ -55,7 +55,7 @@ public static String stripAllTags(String unsafe) {
             return null;
         }
 
-        return Parser.unescapeEntities(Jsoup.clean(unsafe, Whitelist.none()), true);
+        return Parser.unescapeEntities(Jsoup.clean(unsafe, Safelist.none()), true);
 
     }