Merge branch 'develop' into 4579-solr-7.3.0

conf/docker/build.sh

@@ -1,10 +1,15 @@
 #!/bin/sh
 # Creates images and pushes them to Docker Hub.
-# The "latest" tag should be relatively stable. Don't push breaking changes there.
+# The "latest" tag under "iqss" should be relatively stable. Don't push breaking changes there.
 # None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040
-# Push to custom tags or tags based on branch names to iterate on the images.
+# To interate on images, push to custom tags or tags based on branch names or a non-iqss Docker Hub org/username.
+# Docker Hub organization or username
+HUBORG=iqss
+# The most stable tag we have.
+STABLE=latest
+#FIXME: Use a real flag/argument parser. download-files.sh uses "getopts" for example.
 if [ -z "$1" ]; then
-  echo "No argument supplied. Please specify \"branch\" or \"custom my-custom-tag\" for experiments or \"stable\" if your change won't break anything."
+  echo "No argument supplied. For experiments, specify \"branch\" or \"custom my-custom-tag\" or \"huborg <USERNAME/ORG>\". Specify \"stable\" to push to the \"$STABLE\" tag under \"$HUBORG\" if your change won't break anything."
   exit 1
 fi
 
@@ -14,23 +19,32 @@ if [ "$1" == 'branch' ]; then
   TAG=$GIT_BRANCH
 elif [ "$1" == 'stable' ]; then
   echo "We'll push a tag to the most stable tag (which isn't saying much!)."
-  TAG=kick-the-tires
+  TAG=$STABLE
 elif [ "$1" == 'custom' ]; then
-  if [ -z "$1" ]; then
-    echo "You must provide a custom tag as the second argument."
+  if [ -z "$2" ]; then
+    echo "You must provide a custom tag as the second argument. Something other than \"$STABLE\"."
     exit 1
   else
     echo "We'll push a custom tag."
     TAG=$2
   fi
+elif [ "$1" == 'huborg' ]; then
+  if [ -z "$2" ]; then
+    echo "You must provide your Docker Hub organization or username as the second argument. \"$USER\" or whatever."
+    exit 1
+  else
+    HUBORG=$2
+    TAG=$STABLE
+    echo "We'll push to the Docker Hub organization or username you specified: $HUBORG."
+  fi
 else
   echo "Unexpected argument: $1. Exiting. Run with no arguments for help."
   exit 1
 fi
-echo Images will be pushed to Docker Hub with the tag \"$TAG\".
+echo Images will be pushed to Docker Hub org/username \"$HUBORG\" with the tag \"$TAG\".
 # Use "conf" directory as context so we can copy schema.xml into Solr image.
-docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf
-docker push iqss/dataverse-solr:$TAG
+docker build -t $HUBORG/dataverse-solr:$TAG -f solr/Dockerfile ../../conf
+docker push $HUBORG/dataverse-solr:$TAG
 # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip.
 cd ../..
 mvn clean
@@ -58,6 +72,6 @@ if [[ "$?" -ne 0 ]]; then
 fi
 # We'll assume at this point that the download script has been run.
 cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish
-docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish
+docker build -t $HUBORG/dataverse-glassfish:$TAG dataverse-glassfish
 # FIXME: Check the output of `docker build` and only push on success.
-docker push iqss/dataverse-glassfish:$TAG
+docker push $HUBORG/dataverse-glassfish:$TAG

conf/docker/dataverse-glassfish/default.config

@@ -3,7 +3,7 @@ GLASSFISH_DIRECTORY	/usr/local/glassfish4
 ADMIN_EMAIL	 
 MAIL_SERVER	mail.hmdc.harvard.edu
 POSTGRES_ADMIN_PASSWORD	secret
-POSTGRES_SERVER	dataverse-postgresql-service
+POSTGRES_SERVER	dataverse-postgresql-0.dataverse-postgresql-service
 POSTGRES_PORT	5432
 POSTGRES_DATABASE	dvndb
 POSTGRES_USER	dvnapp

conf/openshift/openshift.json

@@ -42,6 +42,7 @@
         "selector": {
           "name": "iqss-dataverse-postgresql"
         },
+        "clusterIP": "None",
         "ports": [
           {
             "name": "database",
@@ -150,6 +151,10 @@
                   }
                 ],
                 "env": [
+                  {
+                    "name": "POSTGRES_SERVER",
+                    "value": "dataverse-postgresql-0"
+                  },
                   {
                     "name": "POSTGRES_SERVICE_HOST",
                     "value": "dataverse-postgresql-service"
@@ -222,15 +227,17 @@
       }
     },
     {
-      "kind": "DeploymentConfig",
-      "apiVersion": "v1",
+      "kind": "StatefulSet",
+      "apiVersion": "apps/v1beta1",
       "metadata": {
         "name": "dataverse-postgresql",
         "annotations": {
           "template.alpha.openshift.io/wait-for-ready": "true"
         }
       },
       "spec": {
+        "serviceName" : "dataverse-postgresql-service",
+        "replicas" : 1,
         "template": {
           "metadata": {
             "labels": {
@@ -241,7 +248,10 @@
             "containers": [
               {
                 "name": "centos-postgresql-94-centos7",
-                "image": "centos-postgresql-94-centos7",
+                "image": "centos/postgresql-94-centos7",
+                "command": [
+                "sh", "-c", "echo 'Setting up Postgres Master/Slave replication...'; [[ `hostname` =~ -([0-9]+)$ ]] || exit 1; ordinal=${BASH_REMATCH[1]}; if [[ $ordinal -eq 0 ]]; then run-postgresql-master; else run-postgresql-slave; fi;"
+                ],
                 "ports": [
                   {
                     "containerPort": 5432,
@@ -253,10 +263,34 @@
                     "name": "POSTGRESQL_USER",
                     "value": "dvnapp"
                   },
+                  {
+                    "name": "POSTGRESQL_MASTER_USER",
+                    "value": "master"
+                  },
                   {
                     "name": "POSTGRESQL_PASSWORD",
                     "value": "secret"
                   },
+                  {
+                    "name": "POSTGRESQL_MASTER_PASSWORD",
+                    "value": "master"
+                  },
+                  {
+                    "name": "POSTGRESQL_MASTER_SERVICE_NAME",
+                    "value": "dataverse-postgresql-service"
+                  },
+                  {
+                    "name": "POSTGRESQL_MASTER_IP",
+                    "value": "dataverse-postgresql-0.dataverse-postgresql-service"
+                  },
+                  {
+                    "name": "postgresql_master_addr",
+                    "value": "dataverse-postgresql-0.dataverse-postgresql-service"
+                  },
+                  {
+                    "name": "master_fqdn",
+                    "value": "dataverse-postgresql-0.dataverse-postgresql-service"
+                  },
                   {
                     "name": "POSTGRESQL_DATABASE",
                     "value": "dvndb"
@@ -281,6 +315,7 @@
             ]
           }
         },
+
         "strategy": {
           "type": "Rolling",
           "rollingParams": {
@@ -300,17 +335,19 @@
               ],
               "from": {
                 "kind": "ImageStreamTag",
-                "name": "centos-postgresql-94-centos7:latest"
+                "name": "centos/postgresql-94-centos7:latest"
               }
             }
           },
           {
             "type": "ConfigChange"
           }
         ],
-        "replicas": 1,
         "selector": {
-          "name": "iqss-dataverse-postgresql"
+          "name": "iqss-dataverse-postgresql",
+          "matchLabels" : {
+            "name" : "iqss-dataverse-postgresql"
+          }
         }
       }
     },

doc/sphinx-guides/source/developers/big-data-support.rst

@@ -155,7 +155,29 @@ TODO: Document these in the Installation Guide once they're final.
 
 To specify replication sites that appear in rsync URLs:
 
-``curl http://localhost:8080/api/admin/settings/:ReplicationSites -X PUT -d "dv.sbgrid.org:Harvard Medical School:USA,sbgrid.icm.uu.se:Uppsala University:Sweden,sbgrid.ncpss.org:Institut Pasteur de Montevideo:Uruguay,sbgrid.ncpss.org:Shanghai Institutes for Biological Sciences:China"``
+Download :download:`add-storage-site.json <../../../../scripts/api/data/storageSites/add-storage-site.json>` and adjust it to meet your needs. The file should look something like this:
+
+.. literalinclude:: ../../../../scripts/api/data/storageSites/add-storage-site.json
+
+Then add the storage site using curl:
+
+``curl -H "Content-type:application/json" -X POST http://localhost:8080/api/admin/storageSites --upload-file add-storage-site.json``
+
+You make a storage site the primary site by passing "true". Pass "false" to make it not the primary site. (id "1" in the example):
+
+``curl -X PUT -d true http://localhost:8080/api/admin/storageSites/1/primaryStorage``
+
+You can delete a storage site like this (id "1" in the example):
+
+``curl -X DELETE http://localhost:8080/api/admin/storageSites/1``
+
+You can view a single storage site like this: (id "1" in the example):
+
+``curl http://localhost:8080/api/admin/storageSites/1``
+
+You can view all storage site like this:
+
+``curl http://localhost:8080/api/admin/storageSites``
 
 In the GUI, this is called "Local Access". It's where you can compute on files on your cluster.
 

doc/sphinx-guides/source/installation/config.rst

@@ -135,6 +135,7 @@ Here are the configuration options for handles:
 
 - :ref:`dataverse.handlenet.admcredfile`
 - :ref:`dataverse.handlenet.admprivphrase`
+- :ref:`dataverse.handlenet.index`
 
 **Database Settings:**
 
@@ -592,6 +593,10 @@ dataverse.handlenet.admprivphrase
 +++++++++++++++++++++++++++++++++
 This JVM setting is also part of **handles** configuration. The Handle.Net installer lets you choose whether to encrypt the admcredfile private key or not. If you do encrypt it, this is the pass phrase that it's encrypted with. 
 
+dataverse.handlenet.index
++++++++++++++++++++++++++++++++++
+If you want to use different index than the default 300
+
 dataverse.timerServer
 +++++++++++++++++++++
 

scripts/api/data/storageSites/add-storage-site.json

@@ -0,0 +1,6 @@
+{
+  "hostname": "dataverse.librascholar.edu",
+  "name": "LibraScholar, USA",
+  "primaryStorage": true,
+  "transferProtocols": "rsync,posix,globus"
+}

src/main/java/edu/harvard/iq/dataverse/DvObject.java

@@ -110,6 +110,17 @@ public String visit(DataFile df) {
     @Column
     private String storageIdentifier;
 
+//    @OneToMany(mappedBy="dvObject", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
+//    private List<DvObjectStorageLocation> dvObjectStorageLocations;
+//
+//    public List<DvObjectStorageLocation> getDvObjectStorageLocations() {
+//        return dvObjectStorageLocations;
+//    }
+//
+//    public void setDvObjectStorageLocations(List<DvObjectStorageLocation> dvObjectStorageLocations) {
+//        this.dvObjectStorageLocations = dvObjectStorageLocations;
+//    }
+
     /**
      * previewImageAvailable could also be thought of as "thumbnail has been
      * generated. However, were all three thumbnails generated? We might need a

src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java

@@ -101,12 +101,14 @@ public void reRegisterHandle(Dataset dataset) {
             String datasetUrl = getRegistrationUrl(dataset);
 
             logger.info("New registration URL: "+datasetUrl);
+
+            int handlenetIndex = System.getProperty("dataverse.handlenet.index")!=null? Integer.parseInt(System.getProperty("dataverse.handlenet.index")) : 300;
 
             PublicKeyAuthenticationInfo auth = getAuthInfo(dataset.getAuthority());
 
             try {
 
-                AdminRecord admin = new AdminRecord(authHandle.getBytes("UTF8"), 300,
+                AdminRecord admin = new AdminRecord(authHandle.getBytes("UTF8"), handlenetIndex,
                         true, true, true, true, true, true,
                         true, true, true, true, true, true);
 
@@ -144,7 +146,8 @@ public Throwable registerNewHandle(Dataset dataset) {
         String handlePrefix = dataset.getAuthority();
         String handle = getDatasetHandle(dataset);
         String datasetUrl = getRegistrationUrl(dataset);
-
+        int handlenetIndex = System.getProperty("dataverse.handlenet.index")!=null? Integer.parseInt(System.getProperty("dataverse.handlenet.index")) : 300;
+
         logger.info("Creating NEW handle " + handle);
 
         String authHandle = getHandleAuthority(dataset);
@@ -154,7 +157,7 @@ public Throwable registerNewHandle(Dataset dataset) {
 
         try {
 
-            AdminRecord admin = new AdminRecord(authHandle.getBytes("UTF8"), 300,
+            AdminRecord admin = new AdminRecord(authHandle.getBytes("UTF8"), handlenetIndex,
                     true, true, true, true, true, true,
                     true, true, true, true, true, true);
 
@@ -233,13 +236,14 @@ private PublicKeyAuthenticationInfo getAuthInfo(String handlePrefix) {
         logger.log(Level.FINE,"getAuthInfo");
         byte[] key = null;
         String adminCredFile = System.getProperty("dataverse.handlenet.admcredfile");
-
+        int handlenetIndex = System.getProperty("dataverse.handlenet.index")!=null? Integer.parseInt(System.getProperty("dataverse.handlenet.index")) : 300;
+
         key = readKey(adminCredFile);        
         PrivateKey privkey = null;
         privkey = readPrivKey(key, adminCredFile);
         String authHandle =  getHandleAuthority(handlePrefix);
         PublicKeyAuthenticationInfo auth =
-                new PublicKeyAuthenticationInfo(Util.encodeString(authHandle), 300, privkey);
+                new PublicKeyAuthenticationInfo(Util.encodeString(authHandle), handlenetIndex, privkey);
         return auth;
     }
     private String getRegistrationUrl(Dataset dataset) {
@@ -361,15 +365,16 @@ public void deleteIdentifier(Dataset datasetIn) throws Exception  {
         String authHandle = getAuthHandle(datasetIn);
 
         String adminCredFile = System.getProperty("dataverse.handlenet.admcredfile");
-
+        int handlenetIndex = System.getProperty("dataverse.handlenet.index")!=null? Integer.parseInt(System.getProperty("dataverse.handlenet.index")) : 300;
+
         byte[] key = readKey(adminCredFile);
         PrivateKey privkey = readPrivKey(key, adminCredFile);
 
         HandleResolver resolver = new HandleResolver();
         resolver.setSessionTracker(new ClientSessionTracker());
 
         PublicKeyAuthenticationInfo auth =
-                new PublicKeyAuthenticationInfo(Util.encodeString(authHandle), 300, privkey);
+                new PublicKeyAuthenticationInfo(Util.encodeString(authHandle), handlenetIndex, privkey);
 
         DeleteHandleRequest req =
                 new DeleteHandleRequest(Util.encodeString(handle), auth);

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -39,6 +39,7 @@
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
+import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
@@ -230,6 +231,9 @@ String getWrappedMessageWhenJson() {
     @EJB
     DataFileServiceBean fileSvc;
 
+    @EJB
+    StorageSiteServiceBean storageSiteSvc;
+
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     protected EntityManager em;