From 2524928ba4ca624d54a11aa8db34bfbddfc4d03b Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Mon, 23 Oct 2017 16:25:43 -0400 Subject: [PATCH 01/40] supporting solr 6.4.2 #4158 It looks like 7.1 is not much more work, but this is the lowest bar to start with. --- pom.xml | 8 +++--- .../iq/dataverse/AutoCompleteBean.java | 16 ++++++++---- .../harvest/server/OAISetServiceBean.java | 18 ++++++++----- .../iq/dataverse/search/BatchIndex.java | 9 ++++--- .../iq/dataverse/search/IndexServiceBean.java | 22 ++++++++++++---- .../dataverse/search/SearchServiceBean.java | 25 +++++++++++++------ .../search/SolrIndexServiceBean.java | 19 ++++++++++---- 7 files changed, 81 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index 06da0203cf1..aecb13deb59 100644 --- a/pom.xml +++ b/pom.xml @@ -196,14 +196,14 @@ org.apache.solr solr-solrj - 4.6.0 + 6.4.2 - - + org.apache.commons commons-math diff --git a/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java b/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java index 3f3d9e2a84e..4853e8ac2a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java @@ -1,15 +1,18 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; import javax.ejb.EJB; import javax.inject.Named; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +//import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.TermsResponse; import org.apache.solr.client.solrj.response.TermsResponse.Term; @@ -25,11 +28,12 @@ public class AutoCompleteBean implements java.io.Serializable { @EJB SystemConfig systemConfig; - private static SolrServer solrServer; + private static SolrClient solrServer; - public SolrServer getSolrServer(){ + public SolrClient getSolrServer(){ if(solrServer == null){ - solrServer = new HttpSolrServer("http://" + systemConfig.getSolrHostColonPort() + "/solr"); + String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr"; + solrServer = new HttpSolrClient.Builder(urlString).build(); } return solrServer; } @@ -59,6 +63,8 @@ public List complete(String query) { // items = resp.getTerms(solrFieldDatasetFieldDescription); } catch (SolrServerException e) { items = null; + } catch (IOException e) { + logger.warning("Solr query error: " + e); } if (items != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java index 7938a234b40..9b975e27d67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java @@ -31,11 +31,14 @@ import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrServer; -import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; +//import org.apache.solr.client.solrj.impl.HttpSolrServer; +//import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -115,12 +118,13 @@ public OAISet findById(Long id) { return em.find(OAISet.class,id); } - private SolrServer solrServer = null; + private SolrClient solrServer = null; - private SolrServer getSolrServer () { + private SolrClient getSolrServer () { if (solrServer == null) { } - solrServer = new HttpSolrServer("http://" + systemConfig.getSolrHostColonPort() + "/solr"); + String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr"; + solrServer = new HttpSolrClient.Builder(urlString).build(); return solrServer; @@ -266,6 +270,8 @@ public List expandSetQuery(String query) throws OaiSetException { } catch (SolrServerException ex) { logger.fine("Internal Dataverse Search Engine Error"); throw new OaiSetException("Internal Dataverse Search Engine Error"); + } catch (IOException e) { + logger.warning("Solr query error: " + e); } SolrDocumentList docs = queryResponse.getResults(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java b/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java index f29bd6ba52b..783a9fb9f2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java @@ -9,7 +9,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.logging.Logger; -import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrClient; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrInputDocument; @@ -21,10 +22,10 @@ public class BatchIndex { private static final Logger logger = Logger.getLogger(BatchIndex.class.getCanonicalName()); private Collection docs; long batchsize; - private SolrServer server; + private SolrClient server; - public BatchIndex(long size, SolrServer serv){ + public BatchIndex(long size, SolrClient serv){ batchsize = size; docs = new ArrayList<>(); server = serv; @@ -75,7 +76,7 @@ public void setDocs(Collection docs) { /** * @return the server */ - public SolrServer getServer() { + public SolrClient getServer() { return server; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 39b21befd6f..bfa31224761 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -51,10 +51,12 @@ import static javax.ejb.TransactionAttributeType.REQUIRES_NEW; import javax.inject.Named; import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +//import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -108,18 +110,24 @@ public class IndexServiceBean { public static final String HARVESTED = "Harvested"; private String rootDataverseName; private Dataverse rootDataverseCached; - private SolrServer solrServer; + private SolrClient solrServer; @PostConstruct public void init(){ - solrServer = new HttpSolrServer("http://" + systemConfig.getSolrHostColonPort() + "/solr"); + String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr"; + solrServer = new HttpSolrClient.Builder(urlString).build(); + rootDataverseName = findRootDataverseCached().getName(); } @PreDestroy public void close(){ if(solrServer != null){ - solrServer.shutdown(); + try { + solrServer.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } solrServer = null; } } @@ -1387,6 +1395,8 @@ private List findDvObjectInSolrOnly(String type) throws SearchException { queryResponse = solrServer.query(solrQuery); } catch (SolrServerException ex) { throw new SearchException("Error searching Solr for " + type, ex); + } catch (IOException e) { + logger.warning("Solr query error: " + e); } SolrDocumentList results = queryResponse.getResults(); for (SolrDocument solrDocument : results) { @@ -1421,6 +1431,8 @@ private List findFilesOfParentDataset(long parentDatasetId) throws Searc queryResponse = solrServer.query(solrQuery); } catch (SolrServerException ex) { throw new SearchException("Error searching Solr for dataset parent id " + parentDatasetId, ex); + } catch (IOException e) { + logger.warning("Solr query error: " + e); } SolrDocumentList results = queryResponse.getResults(); for (SolrDocument solrDocument : results) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 660b5cc80e7..8155cc27250 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.JsfHelper; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.io.IOException; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; @@ -42,12 +43,15 @@ import javax.ejb.TransactionRolledbackLocalException; import javax.inject.Named; import javax.persistence.NoResultException; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; -import org.apache.solr.client.solrj.SolrServer; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrServer; -import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; +//import org.apache.solr.client.solrj.impl.HttpSolrServer; +//import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; @@ -80,17 +84,22 @@ public class SearchServiceBean { SystemConfig systemConfig; public static final JsfHelper JH = new JsfHelper(); - private SolrServer solrServer; + private SolrClient solrServer; @PostConstruct public void init(){ - solrServer = new HttpSolrServer("http://" + systemConfig.getSolrHostColonPort() + "/solr"); + String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr"; + solrServer = new HttpSolrClient.Builder(urlString).build(); } @PreDestroy public void close(){ if(solrServer != null){ - solrServer.shutdown(); + try { + solrServer.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } solrServer = null; } } @@ -301,7 +310,7 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat // ----------------------------------- // Make the solr query // ----------------------------------- - QueryResponse queryResponse; + QueryResponse queryResponse = null; try { queryResponse = solrServer.query(solrQuery); } catch (RemoteSolrException ex) { @@ -333,6 +342,8 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat return exceptionSolrQueryResponse; } catch (SolrServerException ex) { throw new SearchException("Internal Dataverse Search Engine Error", ex); + } catch (IOException e) { + logger.warning("Solr query error: " + e); } SolrDocumentList docs = queryResponse.getResults(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 1c5085ba63c..9b3157da098 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -28,9 +28,11 @@ import javax.inject.Named; import javax.json.Json; import javax.json.JsonObjectBuilder; -import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrClient; +//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +//import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; @@ -57,17 +59,24 @@ public class SolrIndexServiceBean { public static String numRowsClearedByClearAllIndexTimes = "numRowsClearedByClearAllIndexTimes"; public static String messageString = "message"; - private SolrServer solrServer; + private SolrClient solrServer; @PostConstruct public void init(){ - solrServer = new HttpSolrServer("http://" + systemConfig.getSolrHostColonPort() + "/solr"); + String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr"; + solrServer = new HttpSolrClient.Builder(urlString).build(); + } @PreDestroy public void close(){ if(solrServer != null){ - solrServer.shutdown(); + try { + solrServer.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } + solrServer = null; } } From 79043f7a1c6197331257e7d3634a9df754f4f3e4 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Wed, 25 Oct 2017 18:02:52 -0400 Subject: [PATCH 02/40] Updating libraries that solr uses #4158 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index aecb13deb59..5575054b2ff 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ org.apache.solr solr-solrj - 6.4.2 + 7.1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/7.2.1/solrconfig.xml b/conf/solr/7.2.1/solrconfig.xml new file mode 100644 index 00000000000..d9c0b5170e3 --- /dev/null +++ b/conf/solr/7.2.1/solrconfig.xml @@ -0,0 +1,1368 @@ + + + + + + + + + 7.2.1 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + From 0895e46f8b05833273da6c16a2cc7879954370f1 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 15 Mar 2018 17:01:58 -0400 Subject: [PATCH 07/40] solr script fixes and adding an additional document #4158 --- conf/solr/7.2.1/schema.xml | 2 +- doc/sphinx-guides/source/installation/config.rst | 2 +- scripts/deploy/phoenix.dataverse.org/rebuild | 2 +- scripts/search/add | 4 ++-- scripts/search/clear | 2 +- scripts/search/compare | 2 +- scripts/search/empty-entityid-check | 2 +- scripts/search/files | 1 + 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/conf/solr/7.2.1/schema.xml b/conf/solr/7.2.1/schema.xml index 25069e99db0..ece22675fb3 100644 --- a/conf/solr/7.2.1/schema.xml +++ b/conf/solr/7.2.1/schema.xml @@ -180,7 +180,7 @@ - + diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index f9fed23ecdc..21909e5e533 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -69,7 +69,7 @@ Solr schema.xml ++++++++++ -The :doc:`prerequisites` section explained that Dataverse requires a specific Solr schema file called ``schema.xml`` that can be found in the Dataverse distribution. You should have already replaced the default ``example/solr/collection1/conf/schema.xml`` file that ships with Solr. +The :doc:`prerequisites` section explained that Dataverse requires specific Solr schema & config files called ``schema.xml`` & ``solrconfig.xml`` respectively, that can be found in the Dataverse distribution. You should have already replaced the default ``example/solr/collection1/conf/schema.xml`` & ``example/solr/collection1/conf/solrconfig.xml`` file that ships with Solr. jetty.xml +++++++++ diff --git a/scripts/deploy/phoenix.dataverse.org/rebuild b/scripts/deploy/phoenix.dataverse.org/rebuild index ca92ef59b9e..55c2f26619c 100755 --- a/scripts/deploy/phoenix.dataverse.org/rebuild +++ b/scripts/deploy/phoenix.dataverse.org/rebuild @@ -11,7 +11,7 @@ rm -rf /usr/local/glassfish4/glassfish/domains/domain1/files #psql -U postgres -c "CREATE ROLE dvnapp UNENCRYPTED PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1 psql -U dvnapp -c 'DROP DATABASE "dvndb"' template1 echo $? -curl http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" +curl http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" psql -U dvnapp -c 'CREATE DATABASE "dvndb" WITH OWNER = "dvnapp"' template1 echo $? /usr/local/glassfish4/glassfish/bin/asadmin start-domain diff --git a/scripts/search/add b/scripts/search/add index ec6355ae718..ef3a3f0c815 100755 --- a/scripts/search/add +++ b/scripts/search/add @@ -5,10 +5,10 @@ mkdir -p data #echo "adding to solr..." curl -s http://localhost:8080/api/dataverses > data/dataverses.json -#curl http://localhost:8983/solr/update/json?commit=true -H 'Content-type:application/json' --data-binary @data/dataverses.json +#curl http://localhost:8983/solr/collection1/update/json?commit=true -H 'Content-type:application/json' --data-binary @data/dataverses.json curl -s http://localhost:8080/api/datasets > data/datasets.json -#curl http://localhost:8983/solr/update/json?commit=true -H 'Content-type:application/json' --data-binary @data/datasets.json +#curl http://localhost:8983/solr/collection1/update/json?commit=true -H 'Content-type:application/json' --data-binary @data/datasets.json echo "adding to elasticsearch..." #curl -XPOST http://localhost:9200/dataverse/datasets/1 --data-binary @data/datasets/1.dump diff --git a/scripts/search/clear b/scripts/search/clear index 5908e9f09f9..a2f5f8c8e20 100755 --- a/scripts/search/clear +++ b/scripts/search/clear @@ -1,5 +1,5 @@ #!/bin/sh echo "deleting all data from Solr" -curl http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" +curl http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" # this was for elasticsearch #curl -XDELETE http://localhost:9200/dataverse/ diff --git a/scripts/search/compare b/scripts/search/compare index 9dc8cc68a8f..51a14a5ab3c 100755 --- a/scripts/search/compare +++ b/scripts/search/compare @@ -7,7 +7,7 @@ curl -s "http://localhost:8080/api/search?q=*&key=$FINCHKEY" | jq '.data.fq_actu echo "Database:" scripts/search/dbperms $1 | grep '|' echo "Solr per group docs (old):" -curl -s "http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=entityid%3A$1" | jq '.response.docs[] | {id, name_sort, perms_ss}' +curl -s "http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=entityId%3A$1" | jq '.response.docs[] | {id, name_sort, perms_ss}' echo "Solr permission docs (new):" curl -s "http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=definition_point_dvobject_id_s%3A$1" | jq '.response.docs[] | {definition_point_s,discoverable_by_ss}' echo "Java:" diff --git a/scripts/search/empty-entityid-check b/scripts/search/empty-entityid-check index e9ea02b8f2c..c96a706ebfe 100755 --- a/scripts/search/empty-entityid-check +++ b/scripts/search/empty-entityid-check @@ -1,3 +1,3 @@ #!/bin/sh # see also https://redmine.hmdc.harvard.edu/issues/3809 -curl 'http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=-entityid:*' +curl 'http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=-entityId:*' diff --git a/scripts/search/files b/scripts/search/files index 361c9843458..f8bb6149217 100755 --- a/scripts/search/files +++ b/scripts/search/files @@ -1,3 +1,4 @@ #!/bin/sh curl http://localhost:8080/api/index +# below line is broken curl -s 'http://localhost:8983/solr/collection1/select?rows=100&wt=json&indent=true&q=*&fq=dvtype:files' | jq '.response.docs[] | {name_sort, id, parentid}' From cd78b57b3c3bb682247d1df2e6c0f73d0c1b8cd0 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 15 Mar 2018 19:01:54 -0400 Subject: [PATCH 08/40] More documentation, cleanup, etc #4158 Removed 4.6.0 folder updated a script (more to come) updated documentation --- conf/solr/4.6.0/readme.me | 3 - conf/solr/4.6.0/schema.xml | 1692 ----------------- conf/solr/4.6.0/schema.xml.4.6.0.dist | 1134 ----------- conf/solr/4.6.0/stopwords_en.txt | 53 - conf/solr/4.6.0/stopwords_en.txt.dist | 54 - conf/solr/7.2.1/readme.me | 1 + .../files/etc/systemd/solr.service | 4 +- .../source/installation/config.rst | 15 +- .../source/installation/installation-main.rst | 2 +- 9 files changed, 6 insertions(+), 2952 deletions(-) delete mode 100644 conf/solr/4.6.0/readme.me delete mode 100644 conf/solr/4.6.0/schema.xml delete mode 100644 conf/solr/4.6.0/schema.xml.4.6.0.dist delete mode 100644 conf/solr/4.6.0/stopwords_en.txt delete mode 100644 conf/solr/4.6.0/stopwords_en.txt.dist create mode 100644 conf/solr/7.2.1/readme.me diff --git a/conf/solr/4.6.0/readme.me b/conf/solr/4.6.0/readme.me deleted file mode 100644 index 38c25940db0..00000000000 --- a/conf/solr/4.6.0/readme.me +++ /dev/null @@ -1,3 +0,0 @@ -Please see the dev guide for what to do with Solr config file(s). - -schema.xml.4.6.0.dist is the original schema.xml file that came from the 4.6.0 Solr distribution. It's only included so you can diff the files to see what has changed. diff --git a/conf/solr/4.6.0/schema.xml b/conf/solr/4.6.0/schema.xml deleted file mode 100644 index 323429b62da..00000000000 --- a/conf/solr/4.6.0/schema.xml +++ /dev/null @@ -1,1692 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/4.6.0/schema.xml.4.6.0.dist b/conf/solr/4.6.0/schema.xml.4.6.0.dist deleted file mode 100644 index 4501f64f635..00000000000 --- a/conf/solr/4.6.0/schema.xml.4.6.0.dist +++ /dev/null @@ -1,1134 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/4.6.0/stopwords_en.txt b/conf/solr/4.6.0/stopwords_en.txt deleted file mode 100644 index 4437ee269c3..00000000000 --- a/conf/solr/4.6.0/stopwords_en.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -# Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -not -of -on -or -such -that -the -their -then -there -these -they -this -to -was -will -with diff --git a/conf/solr/4.6.0/stopwords_en.txt.dist b/conf/solr/4.6.0/stopwords_en.txt.dist deleted file mode 100644 index 2c164c0b2a1..00000000000 --- a/conf/solr/4.6.0/stopwords_en.txt.dist +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -# Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -such -that -the -their -then -there -these -they -this -to -was -will -with diff --git a/conf/solr/7.2.1/readme.me b/conf/solr/7.2.1/readme.me new file mode 100644 index 00000000000..4457cf9a7df --- /dev/null +++ b/conf/solr/7.2.1/readme.me @@ -0,0 +1 @@ +Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 46fb6ea8e2a..f6c0ce41955 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,8 +5,8 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = simple -WorkingDirectory = /usr/local/solr/example -ExecStart = /usr/bin/java -jar -server /usr/local/solr/example/start.jar +WorkingDirectory = /usr/local/solr-7.2.1 +ExecStart = bin/solr start Restart=on-failure [Install] diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 21909e5e533..0e560442826 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -74,22 +74,11 @@ The :doc:`prerequisites` section explained that Dataverse requires specific Solr jetty.xml +++++++++ -Stop Solr and edit ``solr-4.6.0/example/etc/jetty.xml`` to add a line having to do with ``requestHeaderSize`` as follows: +Stop Solr and edit ``solr-7.2.1/server/etc/jetty.xml`` , changing ``requestHeaderSize`` as follows: .. code-block:: xml - - - - - - 50000 - 1500 - false - 102400 - - - + Without this ``requestHeaderSize`` line in place, which increases the default size, it will appear that no data has been added to your Dataverse installation and ``WARN org.eclipse.jetty.http.HttpParser – HttpParser Full for /127.0.0.1:8983`` will appear in the Solr log. See also https://support.lucidworks.com/hc/en-us/articles/201424796-Error-when-submitting-large-query-strings- diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 12a2eeefee9..3ea8c90ce52 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -198,7 +198,7 @@ Clear Solr The database is fresh and new but Solr has stale data it in. Clear it out with this command: -``curl http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}"`` +``curl http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}"`` Deleting Uploaded Files From 265e2e954d7bf4df4de97df825434bf84e1aa745 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 12:14:37 -0400 Subject: [PATCH 09/40] Updated init, docker, vagrant scripts for solr 7.2.1 #4158 These still need testing --- conf/docker/solr/entrypoint.sh | 4 ++-- conf/vagrant/etc/init.d/solr | 11 +++++------ .../_static/installation/files/etc/init.d/solr | 11 +++++------ .../installation/files/etc/systemd/solr.service | 2 +- .../source/developers/dev-environment.rst | 1 + scripts/vagrant/setup-solr.sh | 13 +++++++++---- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh index 7fd8d6380c2..8b9b25fd299 100755 --- a/conf/docker/solr/entrypoint.sh +++ b/conf/docker/solr/entrypoint.sh @@ -1,8 +1,8 @@ #!/bin/bash if [ "$1" = 'solr' ]; then - cd /usr/local/solr-4.6.0/example/ - java -jar start.jar + cd /usr/local/solr-7.2.1/ + bin/solr start elif [ "$1" = 'usage' ]; then echo 'docker run -d iqss/dataverse-solr solr' else diff --git a/conf/vagrant/etc/init.d/solr b/conf/vagrant/etc/init.d/solr index 5044a1b1e62..853a272493a 100755 --- a/conf/vagrant/etc/init.d/solr +++ b/conf/vagrant/etc/init.d/solr @@ -6,21 +6,20 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr-4.6.0/example" -JAVA_OPTIONS="-Xmx1024m -DSTOP.PORT=8079 -DSTOP.KEY=mustard -jar start.jar" -LOG_FILE="/var/log/solr.log" -JAVA="/usr/bin/java" +SOLR_DIR="/usr/local/solr-7.2.1" +SOLR_COMMAND="bin/solr" +SOLR_ARGS="-m 1g" case $1 in start) echo "Starting Solr" cd $SOLR_DIR - $JAVA $JAVA_OPTIONS 2> $LOG_FILE & + $SOLR_COMMAND start $SOLR_ARGS ;; stop) echo "Stopping Solr" cd $SOLR_DIR - $JAVA $JAVA_OPTIONS --stop + $SOLR_COMMAND stop $SOLR_ARGS ;; restart) $0 stop diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 5044a1b1e62..853a272493a 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -6,21 +6,20 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr-4.6.0/example" -JAVA_OPTIONS="-Xmx1024m -DSTOP.PORT=8079 -DSTOP.KEY=mustard -jar start.jar" -LOG_FILE="/var/log/solr.log" -JAVA="/usr/bin/java" +SOLR_DIR="/usr/local/solr-7.2.1" +SOLR_COMMAND="bin/solr" +SOLR_ARGS="-m 1g" case $1 in start) echo "Starting Solr" cd $SOLR_DIR - $JAVA $JAVA_OPTIONS 2> $LOG_FILE & + $SOLR_COMMAND start $SOLR_ARGS ;; stop) echo "Stopping Solr" cd $SOLR_DIR - $JAVA $JAVA_OPTIONS --stop + $SOLR_COMMAND stop $SOLR_ARGS ;; restart) $0 stop diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index f6c0ce41955..80d82a2c3c4 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -6,7 +6,7 @@ After = syslog.target network.target remote-fs.target nss-lookup.target User = solr Type = simple WorkingDirectory = /usr/local/solr-7.2.1 -ExecStart = bin/solr start +ExecStart = bin/solr start -m 1g Restart=on-failure [Install] diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index c66b70dff67..f3706afd29e 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -152,6 +152,7 @@ Download solr-7.2.1.tgz from http://archive.apache.org/dist/lucene/solr/7.2.1/so - ``mv _default collection1`` - ``cp ~/NetBeansProjects/dataverse/conf/solr/7.2.1/schema.xml solr-7.2.1/server/solr/collection1/conf/schema.xml`` - ``cp ~/NetBeansProjects/dataverse/conf/solr/7.2.1/solrconfig.xml solr-7.2.1/server/solr/collection1/conf/solrconfig.xml`` +- ``cd ~/solr/solr-7.2.1`` - ``bin/solr start`` - ``bin/solr create_core -c collection1 -d server/solr/collection1/conf/`` diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index f4a5bd2fe29..47315922cea 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -4,7 +4,12 @@ GLASSFISH_USER=glassfish GLASSFISH_USER_HOME=~glassfish SOLR_HOME=$GLASSFISH_USER_HOME/solr su $GLASSFISH_USER -s /bin/sh -c "mkdir $SOLR_HOME" -su $GLASSFISH_USER -s /bin/sh -c "cp /downloads/solr-4.6.0.tgz $SOLR_HOME" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-4.6.0.tgz" -su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/4.6.0/schema.xml $SOLR_HOME/solr-4.6.0/example/solr/collection1/conf/schema.xml" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-4.6.0/example && java -jar start.jar &" +su $GLASSFISH_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME" +su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz" +su $GLASSFISH_USER -s /bin/sh -c "cd solr-7.2.1/server/solr" +su $GLASSFISH_USER -s /bin/sh -c "cp -r configsets\_default ." +su $GLASSFISH_USER -s /bin/sh -c "mv _default collection1" +su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" +su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" +su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start" +su $GLASSFISH_USER -s /bin/sh -c "bin/solr create_core -c collection1 -d server/solr/collection1/conf/" From 43a7a175074fd4c76794ffbae55c5301c55ccaa9 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 12:38:16 -0400 Subject: [PATCH 10/40] Many more solr script files updated to 7.2.1 #4158 Hopefully got them all. Still untested --- PULL_REQUEST_TEMPLATE.md | 2 +- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/1prep.sh | 3 ++- conf/docker-aio/c7.dockerfile | 4 ++-- conf/docker-aio/entrypoint.bash | 4 ++-- conf/docker-aio/readme.txt | 2 +- conf/docker/solr/Dockerfile | 22 +++++++++++++--------- downloads/.gitignore | 2 +- downloads/download.sh | 2 +- scripts/installer/Makefile | 7 ++++++- scripts/installer/README.txt | 3 ++- scripts/vagrant/setup.sh | 2 +- 12 files changed, 34 insertions(+), 23 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 37f2ef7bd68..3033524b4f8 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -16,6 +16,6 @@ Welcome! New contributors should at least glance at [CONTRIBUTING.md](/CONTRIBUT [tests]: http://guides.dataverse.org/en/latest/developers/testing.html [SQL updates]: https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades -[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/4.6.0/schema.xml +[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/7.2.1/schema.xml [docs]: http://guides.dataverse.org/en/latest/developers/documentation.html [branch]: http://guides.dataverse.org/en/latest/developers/branching-strategy.html diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 056bec03eb6..4e800d7d3c0 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -17,11 +17,11 @@ if [ ! -e dv/deps/glassfish4dv.tgz ]; then # assuming that folks usually have /tmp auto-clean as needed fi -if [ ! -e dv/deps/solr-4.6.0dv.tgz ]; then +if [ ! -e dv/deps/solr-7.2.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... cd dv/deps/ - wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz -O solr-4.6.0dv.tgz + wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz -O solr-7.2.1dv.tgz cd ../../ fi diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh index 88642a89d78..946455e19e8 100755 --- a/conf/docker-aio/1prep.sh +++ b/conf/docker-aio/1prep.sh @@ -6,7 +6,8 @@ export LANG="en_US.UTF-8" # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world mkdir -p testdata/doc/sphinx-guides/source/_static/util/ -cp ../solr/4.6.0/schema.xml testdata/ +cp ../solr/7.2.1/schema.xml testdata/ +cp ../solr/7.2.1/solrconfig.xml testdata/ cp ../jhove/jhove.conf testdata/ cd ../../ cp -r scripts conf/docker-aio/testdata/ diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile index cf589ff1525..c40d366e644 100644 --- a/conf/docker-aio/c7.dockerfile +++ b/conf/docker-aio/c7.dockerfile @@ -6,14 +6,14 @@ RUN yum install -y jq # copy and unpack dependencies (solr, glassfish) COPY dv /tmp/dv COPY testdata/schema.xml /tmp/dv -RUN cd /opt ; tar zxf /tmp/dv/deps/solr-4.6.0dv.tgz +RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.2.1dv.tgz RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data #RUN sudo -u postgres createuser dvnapp # copy configuration related files -RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ ; cp /tmp/dv/schema.xml /opt/solr-4.6.0/example/solr/collection1/conf/schema.xml +RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ ; cp /tmp/dv/schema.xml /opt/solr-7.2.1/server/solr/collection1/conf/schema.xml ; cp /tmp/dv/solrconfig.xml /opt/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml # skipping glassfish user and solr user (run both as root) diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index fe8b3030677..8ca37c12dfc 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -1,8 +1,8 @@ #!/usr/bin/env bash sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data & -cd /opt/solr-4.6.0/example/ -java -DSTOP.PORT=8079 -DSTOP.KEY=a09df7a0d -jar start.jar & +cd /opt/solr-7.2.1/ +bin/solr stop & cd /opt/glassfish4 bin/asadmin start-domain diff --git a/conf/docker-aio/readme.txt b/conf/docker-aio/readme.txt index 7ef7247c23f..f071f0ad663 100644 --- a/conf/docker-aio/readme.txt +++ b/conf/docker-aio/readme.txt @@ -4,7 +4,7 @@ Could be potentially usable for normal development as well. Initial setup (aka - do once): -- Do surgery on glassfish4 and solr4.6.0 following guides, place results in `conf/docker-aio/dv/deps` as `glassfish4dv.tgz` and `solr-4.6.0dv.tgz` respectively. If you `cd conf/docker-aio` and run `./0prep_deps.sh` these tarballs will be constructed for you. +- Do surgery on glassfish4 and solr7.2.1 following guides, place results in `conf/docker-aio/dv/deps` as `glassfish4dv.tgz` and `solr-7.2.1dv.tgz` respectively. If you `cd conf/docker-aio` and run `./0prep_deps.sh` these tarballs will be constructed for you. Per-build: - `cd conf/docker-aio`, and run `1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile index 99114ce6a6d..79304fdcaa3 100644 --- a/conf/docker/solr/Dockerfile +++ b/conf/docker/solr/Dockerfile @@ -3,21 +3,25 @@ MAINTAINER Dataverse (support@dataverse.org) RUN yum install -y wget unzip perl git java-1.8.0-openjdk-devel postgresql.x86_64 -# Install Solr 4.6.0 +# Install Solr 7.2.1 # The context of the build is the "conf" directory. -COPY solr/4.6.0/schema.xml /tmp +COPY solr/7.2.1/schema.xml /tmp +COPY solr/7.2.1/solrconfig.xml /tmp -RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz && \ - tar xvzf solr-4.6.0.tgz && \ - mv solr-4.6.0 /usr/local/ && \ - cd /usr/local/solr-4.6.0/example/solr/collection1/conf/ && \ + +RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz && \ + tar xvzf solr-7.2.1.tgz && \ + mv solr-7.2.1 /usr/local/ && \ + cd /usr/local/solr-7.2.1/server/solr/collection1/conf/ && \ mv schema.xml schema.xml.backup && \ + mv solrconfig.xml solrconfig.xml.backup && \ cp /tmp/schema.xml . && \ - rm /tmp/solr-4.6.0.tgz + cp /tmp/solrconfig.xml . && \ + rm /tmp/solr-7.2.1.tgz -RUN ln -s /usr/local/solr-4.6.0/example/logs /var/log/solr +RUN ln -s /usr/local/solr-7.2.1/server/logs /var/log/solr -VOLUME /usr/local/solr-4.6.0/example/solr/collection1/data +VOLUME /usr/local/solr-7.2.1/server/solr/collection1/data EXPOSE 8983 diff --git a/downloads/.gitignore b/downloads/.gitignore index 21c1afb26d9..f481e0064a9 100644 --- a/downloads/.gitignore +++ b/downloads/.gitignore @@ -1,4 +1,4 @@ glassfish-4.1.zip -solr-4.6.0.tgz +solr-7.2.1.tgz weld-osgi-bundle-2.2.10.Final-glassfish4.jar schemaSpy_5.0.0.jar diff --git a/downloads/download.sh b/downloads/download.sh index 176ac5f6dea..b858e264b72 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh curl -L -O http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip -curl -L -O https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz +curl -L -O https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz curl -L -O http://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 046e6cb73cd..ad6fa7a6d12 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -70,7 +70,12 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd @mkdir -p ${INSTALLER_ZIP_DIR} /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/4.6.0/schema.xml +${SOLR_SCHEMA}: ../../conf/solr/7.2.1/schema.xml @echo copying Solr schema file @mkdir -p ${INSTALLER_ZIP_DIR} /bin/cp ../../conf/solr/4.6.0/schema.xml ${INSTALLER_ZIP_DIR} + +${SOLR_SCHEMA}: ../../conf/solr/7.2.1/solrconfig.xml + @echo copying Solr config file + @mkdir -p ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/4.6.0/solrconfig.xml ${INSTALLER_ZIP_DIR} \ No newline at end of file diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt index fb41f4fbf85..848e6ee0f31 100644 --- a/scripts/installer/README.txt +++ b/scripts/installer/README.txt @@ -40,6 +40,7 @@ from conf/jhove: jhove.conf -SOLR schema file, from conf/solr/4.6.0: +SOLR schema and config files, from conf/solr/7.2.1: schema.xml +solrconfig.xml diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 0ab2daf93a2..f84292ed33d 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -29,7 +29,7 @@ echo "Ensuring Unix user '$GLASSFISH_USER' exists" useradd $GLASSFISH_USER || : DOWNLOAD_DIR='/dataverse/downloads' GLASSFISH_ZIP="$DOWNLOAD_DIR/glassfish-4.1.zip" -SOLR_TGZ="$DOWNLOAD_DIR/solr-4.6.0.tgz" +SOLR_TGZ="$DOWNLOAD_DIR/solr-7.2.1.tgz" WELD_PATCH="$DOWNLOAD_DIR/weld-osgi-bundle-2.2.10.Final-glassfish4.jar" if [ ! -f $GLASSFISH_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $GLASSFISH_ZIP or $SOLR_TGZ! Running download script...." From 97243cd24c6620f487db8585cccb2321ef038e85 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 12:45:43 -0400 Subject: [PATCH 11/40] Missed one solr 4.6.0 replacement #4158 --- scripts/installer/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index ad6fa7a6d12..0c3d37abd59 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -73,9 +73,9 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${SOLR_SCHEMA}: ../../conf/solr/7.2.1/schema.xml @echo copying Solr schema file @mkdir -p ${INSTALLER_ZIP_DIR} - /bin/cp ../../conf/solr/4.6.0/schema.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/7.2.1/schema.xml ${INSTALLER_ZIP_DIR} ${SOLR_SCHEMA}: ../../conf/solr/7.2.1/solrconfig.xml @echo copying Solr config file @mkdir -p ${INSTALLER_ZIP_DIR} - /bin/cp ../../conf/solr/4.6.0/solrconfig.xml ${INSTALLER_ZIP_DIR} \ No newline at end of file + /bin/cp ../../conf/solr/7.2.1/solrconfig.xml ${INSTALLER_ZIP_DIR} \ No newline at end of file From 26ccf5eb58e8761bc904f6137d14f7f17bc79dce Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 13:04:12 -0400 Subject: [PATCH 12/40] Minor solr 7.2.1 clean-up #4158 --- conf/solr/7.2.1/schema.xml | 1292 ++++++++--------- .../source/installation/prerequisites.rst | 2 +- 2 files changed, 647 insertions(+), 647 deletions(-) diff --git a/conf/solr/7.2.1/schema.xml b/conf/solr/7.2.1/schema.xml index ece22675fb3..45144df7d8d 100644 --- a/conf/solr/7.2.1/schema.xml +++ b/conf/solr/7.2.1/schema.xml @@ -119,621 +119,621 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index b7938477591..16ed1beffc3 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -202,7 +202,7 @@ Solr should be running before the Dataverse installation script is executed. Securing Solr ============= -Solr must be firewalled off from all hosts except the server(s) running Dataverse. Otherwise, any host that can reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://wiki.apache.org/solr/SolrSecurity +Solr must be firewalled off from all hosts except the server(s) running Dataverse. Otherwise, any host that can reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://lucene.apache.org/solr/guide/7_2/securing-solr.html You may want to poke a temporary hole in your firewall to play with the Solr GUI. More information on this can be found in the :doc:`/developers/dev-environment` section of the Developer Guide. From 13ec9c30d29c375c0de548e8034d96ebc7289aa9 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 15:17:05 -0400 Subject: [PATCH 13/40] 4158 Fixed vagrant script, cd-ing solr install when needed #4158 Also confirmed that solr 7.2.1 works in vagrant! --- scripts/vagrant/setup-solr.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 47315922cea..95878d54b71 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -6,10 +6,8 @@ SOLR_HOME=$GLASSFISH_USER_HOME/solr su $GLASSFISH_USER -s /bin/sh -c "mkdir $SOLR_HOME" su $GLASSFISH_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME" su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz" -su $GLASSFISH_USER -s /bin/sh -c "cd solr-7.2.1/server/solr" -su $GLASSFISH_USER -s /bin/sh -c "cp -r configsets\_default ." -su $GLASSFISH_USER -s /bin/sh -c "mv _default collection1" +#su $GLASSFISH_USER -s /bin/sh -c "echo $PWD" +su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r configsets/_default . && mv _default collection1" su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start" -su $GLASSFISH_USER -s /bin/sh -c "bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" From a31a7ec5c3ef85ce37270578c2d87366d72bd823 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 15:35:09 -0400 Subject: [PATCH 14/40] Removing a comment #4158 --- scripts/vagrant/setup-solr.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 95878d54b71..b147fc82bce 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -6,7 +6,6 @@ SOLR_HOME=$GLASSFISH_USER_HOME/solr su $GLASSFISH_USER -s /bin/sh -c "mkdir $SOLR_HOME" su $GLASSFISH_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME" su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz" -#su $GLASSFISH_USER -s /bin/sh -c "echo $PWD" su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r configsets/_default . && mv _default collection1" su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" From 12250e3c8a9607324d5662f015bb22054e621b39 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Fri, 16 Mar 2018 16:46:49 -0400 Subject: [PATCH 15/40] docker copy config fix solr 7.2.1 #4158 --- conf/docker-aio/c7.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile index c40d366e644..f125d49907e 100644 --- a/conf/docker-aio/c7.dockerfile +++ b/conf/docker-aio/c7.dockerfile @@ -6,6 +6,7 @@ RUN yum install -y jq # copy and unpack dependencies (solr, glassfish) COPY dv /tmp/dv COPY testdata/schema.xml /tmp/dv +COPY testdata/solrconfig.xml /tmp/dv RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.2.1dv.tgz RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz From 0ba1059c29413eb73467c2a4368c0f5a65ca405b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 19 Mar 2018 13:06:53 -0400 Subject: [PATCH 16/40] run Solr as dedicated user in Vagrant #4158 --- .gitignore | 1 + scripts/vagrant/setup-solr.sh | 20 ++++++++++---------- scripts/vagrant/setup.sh | 3 +++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 7077e324a2f..1aecf5d09d5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ conf/docker-aio/dv/deps/ conf/docker-aio/dv/install/dvinstall.zip # or copy of test data conf/docker-aio/testdata/ +scripts/installer/default.config diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index b147fc82bce..fe5709ec01b 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -1,12 +1,12 @@ #!/bin/bash echo "Setting up Solr" -GLASSFISH_USER=glassfish -GLASSFISH_USER_HOME=~glassfish -SOLR_HOME=$GLASSFISH_USER_HOME/solr -su $GLASSFISH_USER -s /bin/sh -c "mkdir $SOLR_HOME" -su $GLASSFISH_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r configsets/_default . && mv _default collection1" -su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" -su $GLASSFISH_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" -su $GLASSFISH_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +SOLR_USER=solr +SOLR_HOME=/usr/local/solr +mkdir $SOLR_HOME +chown $SOLR_USER:$SOLR_USER $SOLR_HOME +su $SOLR_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r configsets/_default . && mv _default collection1" +su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" +su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index f84292ed33d..eeef45f20f0 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -27,6 +27,9 @@ chkconfig postgresql on GLASSFISH_USER=glassfish echo "Ensuring Unix user '$GLASSFISH_USER' exists" useradd $GLASSFISH_USER || : +SOLR_USER=solr +echo "Ensuring Unix user '$SOLR_USER' exists" +useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' GLASSFISH_ZIP="$DOWNLOAD_DIR/glassfish-4.1.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-7.2.1.tgz" From 24d7d4def311ad8fa4281e8bc63e6ec3d2d1a77f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 19 Mar 2018 16:36:49 -0400 Subject: [PATCH 17/40] set up Solr init script within Vagrant #4158 --- conf/vagrant/etc/init.d/solr | 8 ++++---- scripts/vagrant/setup-solr.sh | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/conf/vagrant/etc/init.d/solr b/conf/vagrant/etc/init.d/solr index 853a272493a..a5de033dae8 100755 --- a/conf/vagrant/etc/init.d/solr +++ b/conf/vagrant/etc/init.d/solr @@ -1,4 +1,3 @@ -**** #!/bin/sh # Starts, stops, and restarts Apache Solr. @@ -6,20 +5,21 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr-7.2.1" +SOLR_DIR="/usr/local/solr/solr-7.2.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" +SOLR_USER=solr case $1 in start) echo "Starting Solr" cd $SOLR_DIR - $SOLR_COMMAND start $SOLR_ARGS + sudo -u $SOLR_USER $SOLR_COMMAND start $SOLR_ARGS ;; stop) echo "Stopping Solr" cd $SOLR_DIR - $SOLR_COMMAND stop $SOLR_ARGS + sudo -u $SOLR_USER $SOLR_COMMAND stop $SOLR_ARGS ;; restart) $0 stop diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index fe5709ec01b..eabead41721 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -10,3 +10,8 @@ su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r confi su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +cp /dataverse/conf/vagrant/etc/init.d/solr /etc/init.d/solr +chmod 755 /etc/init.d/solr +/etc/init.d/solr stop +/etc/init.d/solr start +chkconfig solr on From 00005bfc32ee8175b587d0806ce88f63f356106d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 15:40:14 -0400 Subject: [PATCH 18/40] remove AutoCompleteBean.java (not used) #108 Sadly, we gave up on getting autocomplete working (removed in 83e8705) so no point in maintaing this AutoCompleteBean.java. #108 was closed a long time ago but maybe some day we can add this feature to Dataverse under a new issue. --- .../iq/dataverse/AutoCompleteBean.java | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java diff --git a/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java b/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java deleted file mode 100644 index d8e630e882a..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/AutoCompleteBean.java +++ /dev/null @@ -1,82 +0,0 @@ -package edu.harvard.iq.dataverse; - -import edu.harvard.iq.dataverse.util.SystemConfig; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; -import javax.ejb.EJB; -import javax.inject.Named; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -//import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -//import org.apache.solr.client.solrj.impl.HttpSolrServer; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.TermsResponse; -import org.apache.solr.client.solrj.response.TermsResponse.Term; - -/** - * @todo: move to service bean - */ -@Named -public class AutoCompleteBean implements java.io.Serializable { - - private static final Logger logger = Logger.getLogger(AutoCompleteBean.class.getCanonicalName()); - - @EJB - SystemConfig systemConfig; - - private static SolrClient solrServer; - - public SolrClient getSolrServer(){ - if(solrServer == null){ - String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr/collection1"; - solrServer = new HttpSolrClient.Builder(urlString).build(); - } - return solrServer; - } - - public List complete(String query) { - List results = new ArrayList<>(); - - solrServer = getSolrServer(); - SolrQuery solrQuery = new SolrQuery(); - solrQuery.setParam("qt", "/terms"); - solrQuery.setTermsLower(query); - solrQuery.setTermsPrefix(query); - // dataverses and files use SearchFields.NAME -// solrQuery.addTermsField(SearchFields.NAME); - solrQuery.addTermsField("text"); -// long datasetFieldDescription = 33L; -// String solrFieldDatasetFieldDescription = datasetFieldService.find(datasetFieldDescription).getSolrField(); -// solrQuery.addTermsField(solrFieldDatasetFieldDescription); - List items = null; - - try { - logger.info("Solr query: " + solrQuery); - QueryResponse qr = solrServer.query(solrQuery); - TermsResponse resp = qr.getTermsResponse(); -// items = resp.getTerms(SearchFields.NAME); - items = resp.getTerms("text"); -// items = resp.getTerms(solrFieldDatasetFieldDescription); - } catch (SolrServerException e) { - items = null; - } catch (IOException e) { - logger.warning("Solr query error: " + e); - } - - if (items != null) { - for (Term term : items) { - logger.info("term: " + term.getTerm()); - results.add(term.getTerm()); - } - } else { - logger.info("no terms found"); - } - - return results; - } - -} From b367e8a68262195b68ec0e57ffd1211de2400770 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 15:54:32 -0400 Subject: [PATCH 19/40] continue to throw OaiSetException on Solr errors #4158 --- .../harvest/server/OAISetServiceBean.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java index 7763be6feef..0b1fbfe383c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java @@ -1,13 +1,6 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.server; import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.harvest.client.ClientHarvestRun; -import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchFields; @@ -33,12 +26,9 @@ import javax.persistence.PersistenceContext; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; -//import org.apache.solr.client.solrj.impl.HttpSolrServer; -//import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -267,11 +257,9 @@ public List expandSetQuery(String query) throws OaiSetException { } logger.fine(error); throw new OaiSetException(error); - } catch (SolrServerException ex) { - logger.fine("Internal Dataverse Search Engine Error"); + } catch (SolrServerException | IOException ex) { + logger.warning("Internal Dataverse Search Engine Error"); throw new OaiSetException("Internal Dataverse Search Engine Error"); - } catch (IOException e) { - logger.warning("Solr query error: " + e); } SolrDocumentList docs = queryResponse.getResults(); From 1245f1243db6b49fae59dc3c8c399eb2adeefbeb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 16:01:25 -0400 Subject: [PATCH 20/40] delete BatchIndex.java (not used) #4158 --- .../iq/dataverse/search/BatchIndex.java | 83 ------------------- 1 file changed, 83 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java diff --git a/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java b/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java deleted file mode 100644 index 783a9fb9f2a..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/search/BatchIndex.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.search; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.logging.Logger; -import org.apache.solr.client.solrj.SolrClient; -//import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; - -/** - * - * @author roberttreacy - */ -public class BatchIndex { -private static final Logger logger = Logger.getLogger(BatchIndex.class.getCanonicalName()); - private Collection docs; - long batchsize; - private SolrClient server; - - - public BatchIndex(long size, SolrClient serv){ - batchsize = size; - docs = new ArrayList<>(); - server = serv; - } - - public void add(SolrInputDocument doc){ - docs.add(doc); - if (docs.size()== batchsize){ - indexDocs(getDocs()); - } - } - - public String indexDocs(Collection docs) { - logger.info("Starting BATCHindex of "+docs.size()+" documents"); - try { - getServer().add(docs); - } catch (SolrServerException | IOException ex) { - return ex.toString(); - } - try { - getServer().commit(); - } catch (SolrServerException | IOException ex) { - return ex.toString(); - } - docs.clear(); - return "indexed " + batchsize + "documents"; - - } - - public String finish(){ - return indexDocs(docs); - } - - /** - * @return the docs - */ - public Collection getDocs() { - return docs; - } - - /** - * @param docs the docs to set - */ - public void setDocs(Collection docs) { - this.docs = docs; - } - - /** - * @return the server - */ - public SolrClient getServer() { - return server; - } - -} From c8d9d15781b589bed9e2983df54a6ef389cbfd50 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 16:07:19 -0400 Subject: [PATCH 21/40] continue to throw SolrServerException on Solr error #4158 Plus other minor cleanup. --- .../iq/dataverse/search/IndexServiceBean.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 1ec091e26f7..9ebc6450e1a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -17,13 +17,12 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.util.StringUtil; -import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; +import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; import java.sql.Timestamp; @@ -53,10 +52,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; -//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; -//import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -122,7 +119,7 @@ public void init(){ @PreDestroy public void close(){ - if(solrServer != null){ + if (solrServer != null) { try { solrServer.close(); } catch (IOException e) { @@ -1393,10 +1390,8 @@ private List findDvObjectInSolrOnly(String type) throws SearchException { QueryResponse queryResponse = null; try { queryResponse = solrServer.query(solrQuery); - } catch (SolrServerException ex) { + } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr for " + type, ex); - } catch (IOException e) { - logger.warning("Solr query error: " + e); } SolrDocumentList results = queryResponse.getResults(); for (SolrDocument solrDocument : results) { @@ -1429,10 +1424,8 @@ private List findFilesOfParentDataset(long parentDatasetId) throws Searc QueryResponse queryResponse = null; try { queryResponse = solrServer.query(solrQuery); - } catch (SolrServerException ex) { + } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr for dataset parent id " + parentDatasetId, ex); - } catch (IOException e) { - logger.warning("Solr query error: " + e); } SolrDocumentList results = queryResponse.getResults(); for (SolrDocument solrDocument : results) { From 77b5141d4c625da2e5f81e097995ec630c76860d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 16:11:22 -0400 Subject: [PATCH 22/40] continue to throw SearchException on Solr error #4158 --- .../iq/dataverse/search/SearchServiceBean.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index ad568edccd9..52b6d149aa2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -1,21 +1,18 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFacet; -import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.JsfHelper; @@ -28,7 +25,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -46,12 +42,9 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; -//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; -//import org.apache.solr.client.solrj.impl.HttpSolrServer; -//import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; @@ -93,8 +86,8 @@ public void init(){ } @PreDestroy - public void close(){ - if(solrServer != null){ + public void close() { + if (solrServer != null) { try { solrServer.close(); } catch (IOException e) { @@ -340,10 +333,8 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat exceptionSolrQueryResponse.setTypeFacetCategories(exceptionFacetCategoryList); exceptionSolrQueryResponse.setSpellingSuggestionsByToken(emptySpellingSuggestion); return exceptionSolrQueryResponse; - } catch (SolrServerException ex) { + } catch (SolrServerException | IOException ex) { throw new SearchException("Internal Dataverse Search Engine Error", ex); - } catch (IOException e) { - logger.warning("Solr query error: " + e); } SolrDocumentList docs = queryResponse.getResults(); From f414d75c16137149e7105b6fcb60b024a19688a1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 20 Mar 2018 16:15:47 -0400 Subject: [PATCH 23/40] minor style fixes #4158 --- .../harvard/iq/dataverse/search/SearchServiceBean.java | 2 +- .../iq/dataverse/search/SolrIndexServiceBean.java | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 52b6d149aa2..af2b852513a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -80,7 +80,7 @@ public class SearchServiceBean { private SolrClient solrServer; @PostConstruct - public void init(){ + public void init() { String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr/collection1"; solrServer = new HttpSolrClient.Builder(urlString).build(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 2077342957b..707baff274a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -29,10 +29,8 @@ import javax.json.Json; import javax.json.JsonObjectBuilder; import org.apache.solr.client.solrj.SolrClient; -//import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; -//import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; @@ -62,21 +60,21 @@ public class SolrIndexServiceBean { private SolrClient solrServer; @PostConstruct - public void init(){ + public void init() { String urlString = "http://" + systemConfig.getSolrHostColonPort() + "/solr/collection1"; solrServer = new HttpSolrClient.Builder(urlString).build(); } @PreDestroy - public void close(){ - if(solrServer != null){ + public void close() { + if (solrServer != null) { try { solrServer.close(); } catch (IOException e) { logger.severe("Solr closing error: " + e); } - + solrServer = null; } } From 27dc0a851c5836079377913f9c1e862571575e8c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Mar 2018 12:33:16 -0400 Subject: [PATCH 24/40] get Solr running in Docker all in one #4158 --- conf/docker-aio/1prep.sh | 2 +- conf/docker-aio/c7.dockerfile | 5 ++++- conf/docker-aio/entrypoint.bash | 5 ++++- conf/docker-aio/readme.txt | 5 ++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh index 946455e19e8..8688d16ade6 100755 --- a/conf/docker-aio/1prep.sh +++ b/conf/docker-aio/1prep.sh @@ -16,7 +16,7 @@ cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/test # not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops mvn clean -./scripts/database/homebrew/custom-build-number +./scripts/installer/custom-build-number mvn package cd scripts/installer make clean diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile index f125d49907e..930ac6890ea 100644 --- a/conf/docker-aio/c7.dockerfile +++ b/conf/docker-aio/c7.dockerfile @@ -14,7 +14,10 @@ RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data #RUN sudo -u postgres createuser dvnapp # copy configuration related files -RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ ; cp /tmp/dv/schema.xml /opt/solr-7.2.1/server/solr/collection1/conf/schema.xml ; cp /tmp/dv/solrconfig.xml /opt/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml +RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ +RUN cp -r /opt/solr-7.2.1/server/solr/configsets/_default /opt/solr-7.2.1/server/solr/collection1 +RUN cp /tmp/dv/schema.xml /opt/solr-7.2.1/server/solr/collection1/conf/schema.xml +RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml # skipping glassfish user and solr user (run both as root) diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index 8ca37c12dfc..ebbe20d7ab2 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -2,8 +2,11 @@ sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data & cd /opt/solr-7.2.1/ -bin/solr stop & +# TODO: Run Solr as non-root and remove "-force". +bin/solr start -force +bin/solr create_core -c collection1 -d server/solr/collection1/conf -force +# TODO: Run Glassfish as non-root. cd /opt/glassfish4 bin/asadmin start-domain sleep infinity diff --git a/conf/docker-aio/readme.txt b/conf/docker-aio/readme.txt index f071f0ad663..3c7c5228aef 100644 --- a/conf/docker-aio/readme.txt +++ b/conf/docker-aio/readme.txt @@ -2,12 +2,11 @@ first pass docker all-in-one image, intended for running integration tests again Could be potentially usable for normal development as well. - Initial setup (aka - do once): -- Do surgery on glassfish4 and solr7.2.1 following guides, place results in `conf/docker-aio/dv/deps` as `glassfish4dv.tgz` and `solr-7.2.1dv.tgz` respectively. If you `cd conf/docker-aio` and run `./0prep_deps.sh` these tarballs will be constructed for you. +- `cd conf/docker-aio` and run `./0prep_deps.sh` to created Glassfish and Solr tarballs in `conf/docker-aio/dv/deps`. Per-build: -- `cd conf/docker-aio`, and run `1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file +- `cd conf/docker-aio`, and run `./1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file - build the docker image: `docker build -t dv0 -f c7.dockerfile .` - Run image: `docker run -d -p 8083:8080 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container) From b94795fb3139fe1e6639b5635d999d280c0b4573 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Mar 2018 12:43:58 -0400 Subject: [PATCH 25/40] put solrconfig.xml in dvinstall.zip #4158 --- scripts/installer/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 0c3d37abd59..bf58241ebb5 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -7,6 +7,7 @@ DB_SCRIPT=${INSTALLER_ZIP_DIR}/reference_data.sql JHOVE_CONFIG=${INSTALLER_ZIP_DIR}/jhove.conf JHOVE_SCHEMA=${INSTALLER_ZIP_DIR}/jhoveConfig.xsd SOLR_SCHEMA=${INSTALLER_ZIP_DIR}/schema.xml +SOLR_CONFIG=${INSTALLER_ZIP_DIR}/solrconfig.xml INSTALL_SCRIPT=${INSTALLER_ZIP_DIR}/install installer: dvinstall.zip @@ -14,9 +15,9 @@ installer: dvinstall.zip clean: /bin/rm -rf ${INSTALLER_ZIP_DIR} dvinstall.zip -dvinstall.zip: ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${INSTALL_SCRIPT} +dvinstall.zip: ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${INSTALL_SCRIPT} @echo making installer... - zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${INSTALL_SCRIPT} + zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${INSTALL_SCRIPT} @echo @echo "Done!" @@ -75,7 +76,7 @@ ${SOLR_SCHEMA}: ../../conf/solr/7.2.1/schema.xml @mkdir -p ${INSTALLER_ZIP_DIR} /bin/cp ../../conf/solr/7.2.1/schema.xml ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/7.2.1/solrconfig.xml +${SOLR_CONFIG}: ../../conf/solr/7.2.1/solrconfig.xml @echo copying Solr config file @mkdir -p ${INSTALLER_ZIP_DIR} - /bin/cp ../../conf/solr/7.2.1/solrconfig.xml ${INSTALLER_ZIP_DIR} \ No newline at end of file + /bin/cp ../../conf/solr/7.2.1/solrconfig.xml ${INSTALLER_ZIP_DIR} From 97740ce1d13991fd0785100db060ce244d439556 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Mar 2018 17:07:09 -0400 Subject: [PATCH 26/40] try to get solr 7 working in minishift/openshift #4158 --- conf/docker/build.sh | 3 +++ conf/docker/solr/Dockerfile | 2 +- conf/docker/solr/entrypoint.sh | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/docker/build.sh b/conf/docker/build.sh index a4828ba607f..846b0db758c 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -34,6 +34,9 @@ docker push iqss/dataverse-solr:$TAG # FIXME: Automate the building of dataverse.war and dvinstall.zip. Think about https://github.com/IQSS/dataverse/issues/3974 and https://github.com/IQSS/dataverse/pull/3975 cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war cp ../../scripts/installer/dvinstall.zip dataverse-glassfish +# Note that default.config needs to have the following values: +# POSTGRES_SERVER dataverse-postgresql-service +# SOLR_LOCATION dataverse-solr-service:8983 cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish cp ../../downloads/glassfish-4.1.zip dataverse-glassfish cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile index 79304fdcaa3..8b4d7c6c673 100644 --- a/conf/docker/solr/Dockerfile +++ b/conf/docker/solr/Dockerfile @@ -12,8 +12,8 @@ COPY solr/7.2.1/solrconfig.xml /tmp RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz && \ tar xvzf solr-7.2.1.tgz && \ mv solr-7.2.1 /usr/local/ && \ + cp -r /usr/local/solr-7.2.1/server/solr/configsets/_default /usr/local/solr-7.2.1/server/solr/collection1 && \ cd /usr/local/solr-7.2.1/server/solr/collection1/conf/ && \ - mv schema.xml schema.xml.backup && \ mv solrconfig.xml solrconfig.xml.backup && \ cp /tmp/schema.xml . && \ cp /tmp/solrconfig.xml . && \ diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh index 8b9b25fd299..fd87e374471 100755 --- a/conf/docker/solr/entrypoint.sh +++ b/conf/docker/solr/entrypoint.sh @@ -3,6 +3,8 @@ if [ "$1" = 'solr' ]; then cd /usr/local/solr-7.2.1/ bin/solr start + bin/solr create_core -c collection1 -d server/solr/collection1/conf + sleep infinity elif [ "$1" = 'usage' ]; then echo 'docker run -d iqss/dataverse-solr solr' else From 307ed252595630acd5220acf29c0a345db765e9b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Mar 2018 17:09:11 -0400 Subject: [PATCH 27/40] update URL in "clear out solr" scripts #4158 --- scripts/api/setup-all.sh | 2 +- scripts/search/solr-delete-id | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index bde54c1eed2..56cc24b4af8 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -21,7 +21,7 @@ done command -v jq >/dev/null 2>&1 || { echo >&2 '`jq` ("sed for JSON") is required, but not installed. Download the binary for your platform from http://stedolan.github.io/jq/ and make sure it is in your $PATH (/usr/bin/jq is fine) and executable with `sudo chmod +x /usr/bin/jq`. On Mac, you can install it with `brew install jq` if you use homebrew: http://brew.sh . Aborting.'; exit 1; } echo "deleting all data from Solr" -curl http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" +curl http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" SERVER=http://localhost:8080/api diff --git a/scripts/search/solr-delete-id b/scripts/search/solr-delete-id index 302a84e7196..a6be6919bd7 100755 --- a/scripts/search/solr-delete-id +++ b/scripts/search/solr-delete-id @@ -4,7 +4,7 @@ if [ -z "$1" ]; then exit 1 else echo "Deleting Solr id $1" - OUTPUT=`curl -s http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"id:$1\"}}"` + OUTPUT=`curl -s http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"id:$1\"}}"` # exit code 7 is expected when Solr is down EXIT_CODE=$? #echo $EXIT_CODE From c9c758a8568911e926a42163d40732ef7c371342 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 21 Mar 2018 17:16:04 -0400 Subject: [PATCH 28/40] tweaks to the install guide docs #4158 --- doc/sphinx-guides/source/installation/config.rst | 2 +- doc/sphinx-guides/source/installation/prerequisites.rst | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9dc74611500..020ff171301 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -69,7 +69,7 @@ Solr schema.xml ++++++++++ -The :doc:`prerequisites` section explained that Dataverse requires specific Solr schema & config files called ``schema.xml`` & ``solrconfig.xml`` respectively, that can be found in the Dataverse distribution. You should have already replaced the default ``example/solr/collection1/conf/schema.xml`` & ``example/solr/collection1/conf/solrconfig.xml`` file that ships with Solr. +The :doc:`prerequisites` section explained that Dataverse requires specific Solr schema and config files called ``schema.xml`` and ``solrconfig.xml`` respectively, that can be found in the Dataverse distribution. You should have already replaced the default ``example/solr/collection1/conf/schema.xml`` and ``example/solr/collection1/conf/solrconfig.xml`` file that ships with Solr. jetty.xml +++++++++ diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 16ed1beffc3..3e66b9ae8af 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -174,11 +174,10 @@ Download and install Solr with these commands:: # cd /usr/local/solr-7.2.1/server/solr # cp -r configsets\_default . # mv _default collection1 - # cd collection1/conf/ - # cp -a schema.xml schema.xml.orig + # cd collection1/conf # cp -a solrconfig.xml solrconfig.xml.orig -The reason for backing up the ``schema.xml`` & ``solrconfig.xml`` file is that Dataverse requires a custom Solr configuration and schema to operate. These files are contained in the "dvinstall" zip supplied in each Dataverse release at https://github.com/IQSS/dataverse/releases . Download this zip file, extract ``schema.xml`` & ``solrconfig.xml`` from it, and put it into place (in the same directory as above):: +Dataverse requires a custom Solr configuration and schema to operate and are contained in the "dvinstall.zip" file in each Dataverse release at https://github.com/IQSS/dataverse/releases . Download this zip file, extract ``schema.xml`` & ``solrconfig.xml`` from it, and put it into place (in the same directory as above):: # cp /tmp/schema.xml schema.xml # cp /tmp/solrconfig.xml solrconfig.xml From 94786ff22567e20eca39868b2b1b23bf2a6a563b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 22 Mar 2018 13:00:56 -0400 Subject: [PATCH 29/40] for now, run Solr out of /tmp in openshift #4158 This is an awful hack to just get Solr working at all. We were getting AccessDeniedException when starting it. FIXME added. The Dockerfile was also simplified to remove lines that don't seem to serve any apparent purpose: RUN ln -s /usr/local/solr-7.2.1/server/logs /var/log/solr VOLUME /usr/local/solr-7.2.1/server/solr/collection1/data --- conf/docker/solr/Dockerfile | 15 +-------------- conf/docker/solr/entrypoint.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile index 8b4d7c6c673..6b013a9a001 100644 --- a/conf/docker/solr/Dockerfile +++ b/conf/docker/solr/Dockerfile @@ -8,20 +8,7 @@ RUN yum install -y wget unzip perl git java-1.8.0-openjdk-devel postgresql.x86_6 COPY solr/7.2.1/schema.xml /tmp COPY solr/7.2.1/solrconfig.xml /tmp - -RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz && \ - tar xvzf solr-7.2.1.tgz && \ - mv solr-7.2.1 /usr/local/ && \ - cp -r /usr/local/solr-7.2.1/server/solr/configsets/_default /usr/local/solr-7.2.1/server/solr/collection1 && \ - cd /usr/local/solr-7.2.1/server/solr/collection1/conf/ && \ - mv solrconfig.xml solrconfig.xml.backup && \ - cp /tmp/schema.xml . && \ - cp /tmp/solrconfig.xml . && \ - rm /tmp/solr-7.2.1.tgz - -RUN ln -s /usr/local/solr-7.2.1/server/logs /var/log/solr - -VOLUME /usr/local/solr-7.2.1/server/solr/collection1/data +RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz EXPOSE 8983 diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh index fd87e374471..50c6d288987 100755 --- a/conf/docker/solr/entrypoint.sh +++ b/conf/docker/solr/entrypoint.sh @@ -1,7 +1,16 @@ #!/bin/bash +# FIXME: Don't run Solr out of /tmp! +# Solr is /tmp to avoid AccessDeniedException under Minishift/OpenShift. +SOLR_DIR=/tmp/solr-7.2.1 + if [ "$1" = 'solr' ]; then - cd /usr/local/solr-7.2.1/ + cd /tmp + tar xvfz solr-7.2.1.tgz + cp -r $SOLR_DIR/server/solr/configsets/_default $SOLR_DIR/server/solr/collection1 + cp /tmp/schema.xml $SOLR_DIR/server/solr/collection1/conf + cp /tmp/solrconfig.xml $SOLR_DIR/server/solr/collection1/conf + cd $SOLR_DIR bin/solr start bin/solr create_core -c collection1 -d server/solr/collection1/conf sleep infinity From 106db3ba0d5a1600171a74d8a3e967b522031a18 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 22 Mar 2018 13:40:20 -0400 Subject: [PATCH 30/40] Changing a logging level #4158 --- .../edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 707baff274a..77122b71579 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -72,7 +72,7 @@ public void close() { try { solrServer.close(); } catch (IOException e) { - logger.severe("Solr closing error: " + e); + logger.warning("Solr closing error: " + e); } solrServer = null; From 493badfe268d8b94390f37670e5d5fa6b29797fa Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 22 Mar 2018 15:31:57 -0400 Subject: [PATCH 31/40] restart glassfish to pick up solr server change #4158 #4040 Also add more error checking to build.sh Also track default.config used in minishift/openshift. --- conf/docker/build.sh | 30 +++++++++++++++---- conf/docker/dataverse-glassfish/.gitignore | 1 - .../docker/dataverse-glassfish/default.config | 16 ++++++++++ conf/docker/dataverse-glassfish/entrypoint.sh | 7 +++++ 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 conf/docker/dataverse-glassfish/default.config diff --git a/conf/docker/build.sh b/conf/docker/build.sh index 846b0db758c..95ff41bab66 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -1,6 +1,7 @@ #!/bin/sh # Creates images and pushes them to Docker Hub. -# The "kick-the-tires" tag should be relatively stable. No breaking changes. +# The "latest" tag should be relatively stable. Don't push breaking changes there. +# None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040 # Push to custom tags or tags based on branch names to iterate on the images. if [ -z "$1" ]; then echo "No argument supplied. Please specify \"branch\" or \"custom my-custom-tag\" for experiments or \"stable\" if your change won't break anything." @@ -31,14 +32,31 @@ echo Images will be pushed to Docker Hub with the tag \"$TAG\". docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf docker push iqss/dataverse-solr:$TAG # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. -# FIXME: Automate the building of dataverse.war and dvinstall.zip. Think about https://github.com/IQSS/dataverse/issues/3974 and https://github.com/IQSS/dataverse/pull/3975 +cd ../.. +mvn clean +scripts/installer/custom-build-number +mvn package +cd conf/docker cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war +if [[ "$?" -ne 0 ]]; then + echo "Unable to copy war file into place. Did 'mvn package' work?" + exit 1 +fi +cd ../../scripts/installer +make clean +make +cd ../../conf/docker cp ../../scripts/installer/dvinstall.zip dataverse-glassfish -# Note that default.config needs to have the following values: -# POSTGRES_SERVER dataverse-postgresql-service -# SOLR_LOCATION dataverse-solr-service:8983 -cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish +if [[ "$?" -ne 0 ]]; then + echo "Unable to copy dvinstall.zip file into place. Did 'make' work?" + exit 1 +fi cp ../../downloads/glassfish-4.1.zip dataverse-glassfish +if [[ "$?" -ne 0 ]]; then + echo "Unable to copy Glassfish zip file into place. You must run the download script in that directory once. " + exit 1 +fi +# We'll assume at this point that the download script has been run. cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish # FIXME: Check the output of `docker build` and only push on success. diff --git a/conf/docker/dataverse-glassfish/.gitignore b/conf/docker/dataverse-glassfish/.gitignore index b0e6e38894f..46a20869565 100644 --- a/conf/docker/dataverse-glassfish/.gitignore +++ b/conf/docker/dataverse-glassfish/.gitignore @@ -2,4 +2,3 @@ glassfish-4.1.zip weld-osgi-bundle-2.2.10.Final-glassfish4.jar dvinstall.zip dataverse.war -default.config diff --git a/conf/docker/dataverse-glassfish/default.config b/conf/docker/dataverse-glassfish/default.config new file mode 100644 index 00000000000..c9a7279742d --- /dev/null +++ b/conf/docker/dataverse-glassfish/default.config @@ -0,0 +1,16 @@ +HOST_DNS_ADDRESS localhost +GLASSFISH_DIRECTORY /usr/local/glassfish4 +ADMIN_EMAIL +MAIL_SERVER mail.hmdc.harvard.edu +POSTGRES_ADMIN_PASSWORD secret +POSTGRES_SERVER dataverse-postgresql-service +POSTGRES_PORT 5432 +POSTGRES_DATABASE dvndb +POSTGRES_USER dvnapp +POSTGRES_PASSWORD secret +SOLR_LOCATION dataverse-solr-service:8983 +TWORAVENS_LOCATION NOT INSTALLED +RSERVE_HOST localhost +RSERVE_PORT 6311 +RSERVE_USER rserve +RSERVE_PASSWORD rserve diff --git a/conf/docker/dataverse-glassfish/entrypoint.sh b/conf/docker/dataverse-glassfish/entrypoint.sh index bc1b7eb3f93..8f933519e67 100755 --- a/conf/docker/dataverse-glassfish/entrypoint.sh +++ b/conf/docker/dataverse-glassfish/entrypoint.sh @@ -126,6 +126,13 @@ if [ "$1" = 'dataverse' ]; then # ./setup-irods.sh # fi + # We do change the Solr server in Minishift/OpenShift, which is + # the primary target for all of the work under conf/docker. + echo -e "\n\nRestarting Dataverse in case Solr host was changed..." + /usr/local/glassfish4/glassfish/bin/asadmin stop-domain + sleep 3 + /usr/local/glassfish4/glassfish/bin/asadmin start-domain + echo -e "\n\nDataverse started" sleep infinity From 9ae5a21467aa02fe400456a4250e1c801b3ee4d4 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 22 Mar 2018 16:12:09 -0400 Subject: [PATCH 32/40] re-added solr boosting, removed commented pom import #4518 --- conf/solr/7.2.1/solrconfig.xml | 21 +++++++++++++++++++++ pom.xml | 6 ------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/conf/solr/7.2.1/solrconfig.xml b/conf/solr/7.2.1/solrconfig.xml index d9c0b5170e3..35cab8e3f3d 100644 --- a/conf/solr/7.2.1/solrconfig.xml +++ b/conf/solr/7.2.1/solrconfig.xml @@ -924,6 +924,27 @@ true 10 5 + edismax + +dvName^170 +dvSubject^160 +dvDescription^150 +dvAffiliation^140 +title^130 +subject^120 +keyword^110 +topicClassValue^100 +dsDescriptionValue^90 +authorName^80 +authorAffiliation^70 +publicationCitation^60 +producerName^50 +fileName^40 +fileDescription^30 +variableLabel^20 +variableName^10 +text^1.0 + spellcheck diff --git a/pom.xml b/pom.xml index ad0821ec6c6..e649ebec692 100644 --- a/pom.xml +++ b/pom.xml @@ -198,12 +198,6 @@ solr-solrj 7.2.1 - org.apache.commons commons-math From f90e00abc9e1a76c4c1a88680e9a475e13a6a859 Mon Sep 17 00:00:00 2001 From: matthew-a-dunlap Date: Thu, 22 Mar 2018 16:49:42 -0400 Subject: [PATCH 33/40] Added comment about solr boosting #4158 --- doc/sphinx-guides/source/installation/prerequisites.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 3e66b9ae8af..e8a84db5a24 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -182,12 +182,16 @@ Dataverse requires a custom Solr configuration and schema to operate and are con # cp /tmp/schema.xml schema.xml # cp /tmp/solrconfig.xml solrconfig.xml +Note: Dataverse has customized Solr to boost results that come from certain indexed elements inside Dataverse, for example results matching on the name of a dataset. If you would like to remove this, edit your solrconfig.xml and remove the ```` element and its contents. + With the Dataverse-specific schema in place, you can now start Solr and create the core that will be used to manage search information:: # cd /usr/local/solr-7.2.1 # bin/solr start # bin/solr create_core -c collection1 -d server/solr/collection1/conf/ + + Solr Init Script ================ From 41a7ea9a9e604f37317bc728db8013a785bda95c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 26 Mar 2018 15:55:15 -0400 Subject: [PATCH 34/40] improve Solr docs #4158 --- conf/vagrant/etc/init.d/solr | 34 ------------ .../installation/files/etc/init.d/solr | 8 +-- .../files/etc/systemd/solr.service | 4 +- .../source/installation/config.rst | 21 +------ .../source/installation/prerequisites.rst | 55 +++++++++++-------- scripts/vagrant/setup-solr.sh | 2 +- 6 files changed, 39 insertions(+), 85 deletions(-) delete mode 100755 conf/vagrant/etc/init.d/solr diff --git a/conf/vagrant/etc/init.d/solr b/conf/vagrant/etc/init.d/solr deleted file mode 100755 index a5de033dae8..00000000000 --- a/conf/vagrant/etc/init.d/solr +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -# Starts, stops, and restarts Apache Solr. -# -# chkconfig: 35 92 08 -# description: Starts and stops Apache Solr - -SOLR_DIR="/usr/local/solr/solr-7.2.1" -SOLR_COMMAND="bin/solr" -SOLR_ARGS="-m 1g" -SOLR_USER=solr - -case $1 in - start) - echo "Starting Solr" - cd $SOLR_DIR - sudo -u $SOLR_USER $SOLR_COMMAND start $SOLR_ARGS - ;; - stop) - echo "Stopping Solr" - cd $SOLR_DIR - sudo -u $SOLR_USER $SOLR_COMMAND stop $SOLR_ARGS - ;; - restart) - $0 stop - sleep 1 - $0 start - ;; - *) - echo "Usage: $0 {start|stop|restart}" >&2 - exit 1 - ;; -esac - diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 853a272493a..a5de033dae8 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -1,4 +1,3 @@ -**** #!/bin/sh # Starts, stops, and restarts Apache Solr. @@ -6,20 +5,21 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr-7.2.1" +SOLR_DIR="/usr/local/solr/solr-7.2.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" +SOLR_USER=solr case $1 in start) echo "Starting Solr" cd $SOLR_DIR - $SOLR_COMMAND start $SOLR_ARGS + sudo -u $SOLR_USER $SOLR_COMMAND start $SOLR_ARGS ;; stop) echo "Stopping Solr" cd $SOLR_DIR - $SOLR_COMMAND stop $SOLR_ARGS + sudo -u $SOLR_USER $SOLR_COMMAND stop $SOLR_ARGS ;; restart) $0 stop diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 80d82a2c3c4..a1124e9202f 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,8 +5,8 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = simple -WorkingDirectory = /usr/local/solr-7.2.1 -ExecStart = bin/solr start -m 1g +WorkingDirectory = /usr/local/solr/solr-7.2.1 +ExecStart = /usr/local/solr/solr-7.2.1/bin/solr start -m 1g Restart=on-failure [Install] diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 020ff171301..b606b8d70d6 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4,7 +4,7 @@ Configuration Now that you've successfully logged into Dataverse with a superuser account after going through a basic :doc:`installation-main`, you'll need to secure and configure your installation. -Settings within Dataverse itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. Configuring Solr requires manipulating XML files. +Settings within Dataverse itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. Once you have finished securing and configuring your Dataverse installation, you may proceed to the :doc:`/admin/index` for more information on the ongoing administration of a Dataverse installation. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. @@ -63,25 +63,6 @@ Password complexity rules for "builtin" accounts can be adjusted with a variety - :ref:`:PVGoodStrength` - :ref:`:PVCustomPasswordResetAlertMessage` -Solr ----- - -schema.xml -++++++++++ - -The :doc:`prerequisites` section explained that Dataverse requires specific Solr schema and config files called ``schema.xml`` and ``solrconfig.xml`` respectively, that can be found in the Dataverse distribution. You should have already replaced the default ``example/solr/collection1/conf/schema.xml`` and ``example/solr/collection1/conf/solrconfig.xml`` file that ships with Solr. - -jetty.xml -+++++++++ - -Stop Solr and edit ``solr-7.2.1/server/etc/jetty.xml`` , changing ``requestHeaderSize`` as follows: - -.. code-block:: xml - - - -Without this ``requestHeaderSize`` line in place, which increases the default size, it will appear that no data has been added to your Dataverse installation and ``WARN org.eclipse.jetty.http.HttpParser – HttpParser Full for /127.0.0.1:8983`` will appear in the Solr log. See also https://support.lucidworks.com/hc/en-us/articles/201424796-Error-when-submitting-large-query-strings- - Network Ports ------------- diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index e8a84db5a24..492e57439a6 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -166,49 +166,56 @@ The Dataverse search index is powered by Solr. Installing Solr =============== -Download and install Solr with these commands:: +You should not run Solr as root. Create a user called ``solr`` and a directory to install Solr into:: - # wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz - # tar xvzf solr-7.2.1.tgz - # rsync -auv solr-7.2.1 /usr/local/ - # cd /usr/local/solr-7.2.1/server/solr - # cp -r configsets\_default . - # mv _default collection1 - # cd collection1/conf - # cp -a solrconfig.xml solrconfig.xml.orig + useradd solr + mkdir /usr/local/solr + chown solr:solr /usr/local/solr -Dataverse requires a custom Solr configuration and schema to operate and are contained in the "dvinstall.zip" file in each Dataverse release at https://github.com/IQSS/dataverse/releases . Download this zip file, extract ``schema.xml`` & ``solrconfig.xml`` from it, and put it into place (in the same directory as above):: +Become the ``solr`` user and then download and configure Solr:: - # cp /tmp/schema.xml schema.xml - # cp /tmp/solrconfig.xml solrconfig.xml + su - solr + cd /usr/local/solr + wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz + tar xvzf solr-7.2.1.tgz + cd solr-7.2.1 + cp -r server/solr/configsets/_default server/solr/collection1 -Note: Dataverse has customized Solr to boost results that come from certain indexed elements inside Dataverse, for example results matching on the name of a dataset. If you would like to remove this, edit your solrconfig.xml and remove the ```` element and its contents. +Download "dvinstall.zip" file in each Dataverse release at https://github.com/IQSS/dataverse/releases and unzip it into ``/tmp``. Then copy the files into place:: -With the Dataverse-specific schema in place, you can now start Solr and create the core that will be used to manage search information:: + cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf - # cd /usr/local/solr-7.2.1 - # bin/solr start - # bin/solr create_core -c collection1 -d server/solr/collection1/conf/ +Note: Dataverse has customized Solr to boost results that come from certain indexed elements inside Dataverse, for example results matching on the name of a dataset. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. +Dataverse requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.2.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +With the Dataverse-specific config in place, you can now start Solr and create the core that will be used to manage search information:: + + cd /usr/local/solr/solr-7.2.1 + bin/solr start + bin/solr create_core -c collection1 -d server/solr/collection1/conf/ Solr Init Script ================ -The command above will start Solr which is good for a quick sanity check that Solr accepted the schema file, but letting the system start Solr automatically is recommended. - -- This :download:`Solr Systemd file<../_static/installation/files/etc/systemd/solr.service>` will launch Solr on boot as the solr user for RHEL/CentOS 7 or Ubuntu 16+ systems, or -- For systems using init.d, you may attempt to adjust this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` for your needs or write your own. +For systems running systemd, as root, download :download:`solr.service<../_static/installation/files/etc/systemd/solr.service>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands:: + + cp /tmp/solr.service /usr/lib/systemd/system + systemctl start solr.service + systemctl enable solr.service -Solr should be running before the Dataverse installation script is executed. +For systems using init.d, :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp/``. Then start Solr and configure it to start at boot with the following commands:: + + cp /tmp/solr /etc/init.d + service solr start + chkconfig solr on Securing Solr ============= Solr must be firewalled off from all hosts except the server(s) running Dataverse. Otherwise, any host that can reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://lucene.apache.org/solr/guide/7_2/securing-solr.html -You may want to poke a temporary hole in your firewall to play with the Solr GUI. More information on this can be found in the :doc:`/developers/dev-environment` section of the Developer Guide. - jq -- diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index eabead41721..8cf393fc087 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -10,7 +10,7 @@ su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r confi su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml" su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml" su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" -cp /dataverse/conf/vagrant/etc/init.d/solr /etc/init.d/solr +cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr chmod 755 /etc/init.d/solr /etc/init.d/solr stop /etc/init.d/solr start From 1c46b334a4e843aa827e19b07738621327d6109c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 26 Mar 2018 16:18:38 -0400 Subject: [PATCH 35/40] simplify solr setup steps in dev #4158 --- doc/sphinx-guides/source/developers/dev-environment.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index e01a4584ee2..74bef8bf977 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -123,9 +123,7 @@ To install Solr, execute the following commands: ``cd solr-7.2.1/server/solr`` -``cp -r configsets/_default .`` - -``mv _default collection1`` +``cp -r configsets/_default collection1`` ``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.2.1/schema.xml`` From a7fd43a69ad4260ff40d2910672ead556b9744f2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Mar 2018 10:44:40 -0400 Subject: [PATCH 36/40] replace hard-coded "Match", look up in bundle #4158 "Match" was showing up because the bundle key didn't exist. We are switching away from the deprecated "JsfHelper.localize" method. --- .../iq/dataverse/search/SearchServiceBean.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index af2b852513a..cb0dedf3af6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -15,7 +15,7 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.util.JsfHelper; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; import java.lang.reflect.Field; @@ -76,7 +76,6 @@ public class SearchServiceBean { @EJB SystemConfig systemConfig; - public static final JsfHelper JH = new JsfHelper(); private SolrClient solrServer; @PostConstruct @@ -170,6 +169,7 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat solrQuery.setHighlightSimplePre(""); solrQuery.setHighlightSimplePost(""); Map solrFieldsToHightlightOnMap = new HashMap<>(); + // TODO: Do not hard code "Name" etc as English here. solrFieldsToHightlightOnMap.put(SearchFields.NAME, "Name"); solrFieldsToHightlightOnMap.put(SearchFields.AFFILIATION, "Affiliation"); solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_FRIENDLY, "File Type"); @@ -178,7 +178,7 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_LABEL, "Variable Label"); solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_SEARCHABLE, "File Type"); solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PUBLICATION_DATE, "Publication Date"); - solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, localize("advanced.search.datasets.persistentId")); + solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, BundleUtil.getStringFromBundle("dataset.metadata.persistentId")); /** * @todo Dataverse subject and affiliation should be highlighted but * this is commented out right now because the "friendly" names are not @@ -744,16 +744,6 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat return solrQueryResponse; } - private static String localize(String bundleKey) { - try { - String value = JH.localize(bundleKey); - return value; - } catch (Exception e) { - // can throw MissingResourceException - return "Match"; - } - } - public String getCapitalizedName(String name) { return Character.toUpperCase(name.charAt(0)) + name.substring(1); } From dc8476d41202a0595f1dab0ea32a979e081209a9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Mar 2018 12:03:53 -0400 Subject: [PATCH 37/40] add test to exercise identifier basic search bug #4158 --- .../harvard/iq/dataverse/api/SearchIT.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 191f6242472..f38cf9b8fd8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -576,6 +576,64 @@ public void testDatasetThumbnail() { */ } + @Test + public void testIdentifier() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + System.out.println("id: " + datasetId); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + System.out.println("datasetPid: " + datasetPid); + + Response datasetAsJson = UtilIT.nativeGet(datasetId, apiToken); + datasetAsJson.then().assertThat() + .statusCode(OK.getStatusCode()); + + String identifier = JsonPath.from(datasetAsJson.getBody().asString()).getString("data.identifier"); + System.out.println("identifier: " + identifier); + + Response searchUnpublished = UtilIT.search(identifier, apiToken); + searchUnpublished.prettyPrint(); + searchUnpublished.then().assertThat() + .statusCode(OK.getStatusCode()) + // It's expected that you can't find it because it hasn't been published. + .body("data.total_count", CoreMatchers.equalTo(0)); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response searchTargeted = UtilIT.search("dsPersistentId:" + identifier, apiToken); + searchTargeted.prettyPrint(); + searchTargeted.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.total_count", CoreMatchers.equalTo(1)); + + Response searchUntargeted = UtilIT.search(identifier, apiToken); + searchUntargeted.prettyPrint(); + searchUntargeted.then().assertThat() + .statusCode(OK.getStatusCode()) + // FIXME: Why can't we find the dataset based on just its identifier (basic search)? + // Oddly, you *can* find from Solr directly: curl 'http://localhost:8983/solr/collection1/select?rows=1000000&wt=json&indent=true&hl=true&hl.fl=*&q=JNIUOA' + // Under "highlighting", you see "dsPersistentId":["doi:10.5072/FK2/JNIUOA"] + .body("data.total_count", CoreMatchers.equalTo(0)); + + } + @After public void tearDownDataverse() { File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48"); From d3b721e2d99b8cabd7ab69b21cb401bebc1533f0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Mar 2018 12:42:12 -0400 Subject: [PATCH 38/40] switch from "/spell" to "/select" Solr request handler #4158 Back in 60e640b when I was playing with spelling suggestions from Solr I changed the request handler from "/select" (the default) to "/spell". We didn't have time to fully explore the spelling suggestions feature of Solr during the 4.0 rewrite and the "/spell" request handler seems to be leading to other bugs, such as not being able to search on the "identifier" portion of a DOI (i.e. "JNIUOA") from basic search. In short we are switching to the default request handler for Solr, something I would have done before tagging 4.0 if I had realized I had left the "/spell" request handler in there. --- .../edu/harvard/iq/dataverse/search/SearchServiceBean.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index cb0dedf3af6..8eb57686463 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -205,7 +205,7 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat solrQuery.addHighlightField(solrField); } solrQuery.setParam("fl", "*,score"); - solrQuery.setParam("qt", "/spell"); + solrQuery.setParam("qt", "/select"); solrQuery.setParam("facet", "true"); /** * @todo: do we need facet.query? diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index f38cf9b8fd8..f59265f6862 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -627,10 +627,7 @@ public void testIdentifier() { searchUntargeted.prettyPrint(); searchUntargeted.then().assertThat() .statusCode(OK.getStatusCode()) - // FIXME: Why can't we find the dataset based on just its identifier (basic search)? - // Oddly, you *can* find from Solr directly: curl 'http://localhost:8983/solr/collection1/select?rows=1000000&wt=json&indent=true&hl=true&hl.fl=*&q=JNIUOA' - // Under "highlighting", you see "dsPersistentId":["doi:10.5072/FK2/JNIUOA"] - .body("data.total_count", CoreMatchers.equalTo(0)); + .body("data.total_count", CoreMatchers.equalTo(1)); } From 4ac6613389a76f09651558b6c8c5621d24ff9fae Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Mar 2018 16:44:54 -0400 Subject: [PATCH 39/40] fix wording around dvinstall.zip #4158 --- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 492e57439a6..a6296a5a3ba 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -181,7 +181,7 @@ Become the ``solr`` user and then download and configure Solr:: cd solr-7.2.1 cp -r server/solr/configsets/_default server/solr/collection1 -Download "dvinstall.zip" file in each Dataverse release at https://github.com/IQSS/dataverse/releases and unzip it into ``/tmp``. Then copy the files into place:: +You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf From a088d5efc269d9c56b6fcbe6c2a6b39335480d6a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Mar 2018 18:01:19 -0400 Subject: [PATCH 40/40] fix typo ("download" missing) #4158 --- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index a6296a5a3ba..92188613eab 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -205,7 +205,7 @@ For systems running systemd, as root, download :download:`solr.service<../_stati systemctl start solr.service systemctl enable solr.service -For systems using init.d, :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp/``. Then start Solr and configure it to start at boot with the following commands:: +For systems using init.d, download this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands:: cp /tmp/solr /etc/init.d service solr start