Merge branch 'develop' into 2391-file-edit-tags

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+# RFI Checklist
+
+_**Before** submitting the pull request, fill out sections (1.) Related Issues and (2.) Pull Request Checklist._
+
+### 1. Related Issues
+
+_List and [link](https://guides.github.com/features/issues/#notifications) to the issues in this Pull Request._
+
+- [#### + Title HERE]
+
+---
+### 2. Pull Request Checklist
+
+- [ ]  Functionality completed as described in FRD
+- [ ]  Dependencies, risks, assumptions in FRD addressed
+- [ ]  Unit tests completed
+- [ ]  Deployment requirements identified (e.g., SQL scripts, indexing)
+- [ ]  Documentation completed
+- [ ]  All code checkins completed
+
+---
+### 3. Review Checklist
+
+_**After** the pull request has been submitted, fill out this section._
+
+- [ ]  Code review completed or waived
+- [ ]  Testing requirements completed
+- [ ]  Usability testing completed or waived
+- [ ]  Support testing completed or waived
+- [ ]  Merged with develop branch and resolved conflicts

doc/sphinx-guides/source/_static/docsdataverse_org.css

@@ -5,7 +5,7 @@
  */
 
 body {
-    padding-top: 70px; /* Required padding for .navbar-fixed-top. Remove if using .navbar-static-top. Change if height of navigation changes. */
+    padding-top: 52px; /* Required padding for .navbar-fixed-top. Remove if using .navbar-static-top. Change if height of navigation changes. */
 }
 
 .hero-spacer {

doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish

@@ -0,0 +1,44 @@
+#! /bin/sh
+# chkconfig: 2345 99 01
+# description: GlassFish App Server
+
+set -e
+
+ASADMIN=/usr/local/glassfish4/bin/asadmin
+
+case "$1" in
+  start)
+        echo -n "Starting GlassFish server: glassfish"
+        # Increase file descriptor limit:
+        ulimit -n 32768
+        # Allow "memory overcommit":
+        # (basically, this allows to run exec() calls from inside the
+        # app, without the Unix fork() call physically hogging 2X
+        # the amount of memory glassfish is already using)
+        echo 1 > /proc/sys/vm/overcommit_memory
+
+        #echo
+        #echo "GLASSFISH IS UNDER MAINTENANCE;"
+        #echo "PLEASE DO NOT USE service init script."
+        #echo
+	LANG=en_US.UTF-8; export LANG
+        $ASADMIN start-domain domain1
+        echo "."
+        ;;
+  stop)
+        echo -n "Stopping GlassFish server: glassfish"
+        #echo
+        #echo "GLASSFISH IS UNDER MAINTENANCE;"
+        #echo "PLEASE DO NOT USE service init script."
+        #echo
+
+        $ASADMIN stop-domain domain1
+        echo "."
+        ;;
+
+  *)
+        echo "Usage: /etc/init.d/glassfish {start|stop}"
+        exit 1
+esac
+
+exit 0

doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr

@@ -0,0 +1,35 @@
+****
+#!/bin/sh
+
+# Starts, stops, and restarts Apache Solr.
+#
+# chkconfig: 35 92 08
+# description: Starts and stops Apache Solr
+
+SOLR_DIR="/usr/local/solr-4.6.0/example"
+JAVA_OPTIONS="-Xmx1024m -DSTOP.PORT=8079 -DSTOP.KEY=mustard -jar start.jar"
+LOG_FILE="/var/log/solr.log"
+JAVA="/usr/bin/java"
+
+case $1 in
+    start)
+        echo "Starting Solr"
+        cd $SOLR_DIR
+        $JAVA $JAVA_OPTIONS 2> $LOG_FILE &
+        ;;
+    stop)
+        echo "Stopping Solr"
+        cd $SOLR_DIR
+        $JAVA $JAVA_OPTIONS --stop
+        ;;
+    restart)
+        $0 stop
+        sleep 1
+        $0 start
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|restart}" >&2
+        exit 1
+        ;;
+esac
+

doc/sphinx-guides/source/_static/navbar_from_dataverse_org.js

@@ -16,4 +16,4 @@ function load_navbar(){
         console.log( "error" );
         $('#dv-bs-navbar-details').html('<!-- failed to retrieve navbar -->');
     })
-}
+}

doc/sphinx-guides/source/_static/navbarscroll.js

@@ -0,0 +1,29 @@
+/*  
+    Use to fix hidden section headers behind the navbar when using links with targets
+    See: http://stackoverflow.com/questions/10732690/offsetting-an-html-anchor-to-adjust-for-fixed-header    
+*/
+$jqTheme(document).ready(function() {
+  $jqTheme('a[href*=#]:not([href=#])').on('click', function() {
+    if (location.pathname.replace(/^\//,'') == this.pathname.replace(/^\//,'') 
+&& location.hostname == this.hostname) {
+      var target = $jqTheme(this.hash);
+      target = target.length ? target : $('a.headerlink[href="#' + this.hash.slice(1) +'""]');
+      if (target.length) {
+        $jqTheme('html,body').animate({
+          scrollTop: target.offset().top - 60 //offsets for fixed header
+        }, 1000);
+        return false;
+      }
+    }
+  });
+  //Executed on page load with URL containing an anchor tag.
+  if($jqTheme(location.href.split("#")[1])) {
+      var target = $('#'+location.href.split("#")[1]);
+      if (target.length) {
+        $jqTheme('html,body').animate({
+          scrollTop: target.offset().top - 60 //offset height of header here too.
+        }, 1000);
+        return false;
+      }
+    }
+});

doc/sphinx-guides/source/_templates/layout.html

@@ -9,18 +9,9 @@
   (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
   m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
   })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
-
   ga('create', 'UA-57484542-1', 'auto');
   ga('send', 'pageview');
-
 </script>
-
-
 {% endblock %}
 
-
 {# Import the theme's layout. #}
-
-
-{# Include our new CSS file into existing ones. #}
-{% set css_files = css_files + ['_static/docsdataverse_org.css'] %}

doc/sphinx-guides/source/api/native-api.rst

@@ -99,6 +99,19 @@ Datasets
 * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number.
 * ``x`` same as ``x.0``
 
+
+.. note:: Datasets can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the dataset is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``.
+
+  Example: Getting the dataset whose DOI is *10.5072/FK2/J8SJZB* ::
+
+    GET http://$SERVER/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB
+
+  Getting its draft version::
+
+    GET http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB
+
+
+
 Show the dataset whose id is passed::
 
   GET http://$SERVER/api/datasets/$id?key=$apiKey
@@ -139,6 +152,17 @@ Deletes the draft version of dataset ``$id``. Only the draft version can be dele
 
     DELETE http://$SERVER/api/datasets/$id/versions/:draft?key=$apiKey
 
+Sets the dataset field type to be used as the citation date for the given dataset (if the dataset does not include the dataset field type, the default logic is used). The name of the dataset field type should be sent in the body of the reqeust.
+To revert to the default logic, use ``:publicationDate`` as the ``$datasetFieldTypeName``.
+Note that the dataset field used has to be a date field::
+
+    PUT http://$SERVER/api/datasets/$id/citationdate?key=$apiKey
+    
+Restores the default logic of the field type to be used as the citation date. Same as ``PUT`` with ``:publicationDate`` body::
+    
+    DELETE http://$SERVER/api/datasets/$id/citationdate?key=$apiKey
+
+
 Builtin Users
 ~~~~~
 

doc/sphinx-guides/source/conf.py

@@ -56,16 +56,16 @@
 
 # General information about the project.
 project = u'Dataverse'
-copyright = u'2015, The President & Fellows of Harvard College'
+copyright = u'2016, The President & Fellows of Harvard College'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.3'
+version = '4.2.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.3'
+release = '4.2.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -113,8 +113,8 @@
 html_theme = 'bootstrap'
 
 # Custom Setup: add the CSS file to the app's theme.
-def setup(app):
- app.add_stylesheet( "docsdataverse_org.css" )
+# def setup(app):
+#  app.add_stylesheet( "docsdataverse_org.css" )
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

doc/sphinx-guides/source/index.rst

@@ -3,8 +3,8 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Dataverse 4.3 Guides
-====================
+Dataverse 4.2.4 Guides
+======================
 
 These guides are for the most recent version of Dataverse. For the guides for **version 4.2.3** please go `here <http://guides.dataverse.org/en/4.2.3/>`_.
 

doc/sphinx-guides/source/installation/administration.rst

@@ -1,20 +1,85 @@
 Administration
 ==============
 
+This section focuses on system and database administration tasks. Please see the :doc:`/user/index` for tasks having to do with having the "Admin" role on a dataverse or dataset.
+
 .. contents:: :local:
 
+Solr Search Index
+-----------------
+
+Dataverse requires Solr to be operational at all times. If you stop Solr, you should see a error about this on the home page, which is powered by the search index Solr provides. You set up Solr by following the steps in the :doc:`prerequisites` section and the :doc:`config` section explained how to configure it. This section is about the care and feeding of the search index. PostgreSQL is the "source of truth" and the Dataverse application will copy data from PostgreSQL into Solr. For this reason, the search index can be rebuilt at any time but depending on the amount of data you have, this can be a slow process. You are encouraged to experiment with production data to get a sense of how long a full reindexing will take.
+
+Full Reindex
+++++++++++++
+
+There are two ways to perform a full reindex of the Dataverse search index. Starting with a "clear" ensures a completely clean index but involves downtime. Reindexing in place doesn't involve downtime but does not ensure a completely clean index.
+
+Clear and Reindex
+~~~~~~~~~~~~~~~~~
+
+Clearing Data from Solr
+.......................
+
+Please note that the moment you issue this command, it will appear to end users looking at the home page that all data is gone! This is because the home page is powered by the search index.
+
+``curl http://localhost:8080/api/admin/index/clear``
+
+Start Async Reindex
+...................
+
+Please note that this operation may take hours depending on the amount of data in your system. This known issue is being tracked at https://github.com/IQSS/dataverse/issues/50
+
+``curl http://localhost:8080/api/admin/index``
+
+Reindex in Place
+~~~~~~~~~~~~~~~~
+
+An alternative to completely clearing the search index is to reindex in place.
+
+Clear Index Timestamps
+......................
+
+``curl -X DELETE http://localhost:8080/api/admin/index/timestamps``
+
+Start or Continue Async Reindex
+................................
+
+If indexing stops, this command should pick up where it left off based on which index timestamps have been set, which is why we start by clearing these timestamps above. These timestamps are stored in the ``dvobject`` database table.
+
+``curl http://localhost:8080/api/admin/index/continue``
+
+Glassfish
+---------
+
+``server.log`` is the main place to look when you encounter problems. Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version.
+
+For debugging purposes, you may find it helpful to increase logging levels as mentioned in the :doc:`/developers/debugging` section of the Developer Guide.
+
+This guide has focused on using the command line to manage Glassfish but you might be interested in an admin GUI at http://localhost:4848
+
+Monitoring
+----------
+
+In production you'll want to monitor the usual suspects such as CPU, memory, free disk space, etc.
+
+https://github.com/IQSS/dataverse/issues/2595 contains some information on enabling monitoring of Glassfish, which is disabled by default.
+
+There is a database table called ``actionlogrecord`` that captures events that may be of interest. See https://github.com/IQSS/dataverse/issues/2729 for more discussion around this table.
+
 User Administration
 -------------------
 
-Deleting an API token
-~~~~~~~~~~~~~~~~~~~~~
+There isn't much in the way of user administration tools built in to Dataverse.
+
+Deleting an API Token
++++++++++++++++++++++
 
-If an API token is compromised it should be deleted. Users can generate a new one for themselves, but someone with access to the database can delete tokens as well.
+If an API token is compromised it should be deleted. Users can generate a new one for themselves as explained in the :doc:`/user/account` section of the User Guide, but you may want to preemptively delete tokens from the database.
 
 Using the API token 7ae33670-be21-491d-a244-008149856437 as an example:
 
 ``delete from apitoken where tokenstring = '7ae33670-be21-491d-a244-008149856437';``
 
 You should expect the output ``DELETE 1`` after issuing the command above.
 
-After the API token has been deleted, users can generate a new one per :doc:`/user/account`.