Merge pull request #4520 from IQSS/4158-update-solr

4158 update solr

.gitignore

@@ -41,3 +41,4 @@ conf/docker-aio/dv/deps/
 conf/docker-aio/dv/install/dvinstall.zip
 # or copy of test data
 conf/docker-aio/testdata/
+scripts/installer/default.config

PULL_REQUEST_TEMPLATE.md

@@ -16,6 +16,6 @@ Welcome! New contributors should at least glance at [CONTRIBUTING.md](/CONTRIBUT
 
 [tests]: http://guides.dataverse.org/en/latest/developers/testing.html
 [SQL updates]: https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades
-[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/4.6.0/schema.xml
+[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/7.2.1/schema.xml
 [docs]: http://guides.dataverse.org/en/latest/developers/documentation.html
 [branch]: http://guides.dataverse.org/en/latest/developers/branching-strategy.html

conf/docker-aio/0prep_deps.sh

@@ -17,11 +17,11 @@ if [ ! -e dv/deps/glassfish4dv.tgz ]; then
 	# assuming that folks usually have /tmp auto-clean as needed
 fi
 
-if [ ! -e dv/deps/solr-4.6.0dv.tgz ]; then
+if [ ! -e dv/deps/solr-7.2.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
 	cd dv/deps/
-	wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz -O solr-4.6.0dv.tgz
+	wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz -O solr-7.2.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -6,7 +6,8 @@ export LANG="en_US.UTF-8"
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/4.6.0/schema.xml testdata/
+cp ../solr/7.2.1/schema.xml testdata/
+cp ../solr/7.2.1/solrconfig.xml testdata/
 cp ../jhove/jhove.conf testdata/
 cd ../../
 cp -r scripts conf/docker-aio/testdata/
@@ -15,7 +16,7 @@ cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/test
 
 # not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops
 mvn clean
-./scripts/database/homebrew/custom-build-number
+./scripts/installer/custom-build-number
 mvn package
 cd scripts/installer
 make clean

conf/docker-aio/c7.dockerfile

@@ -6,14 +6,18 @@ RUN yum install -y jq
 # copy and unpack dependencies (solr, glassfish)
 COPY dv /tmp/dv
 COPY testdata/schema.xml /tmp/dv
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-4.6.0dv.tgz 
+COPY testdata/solrconfig.xml /tmp/dv
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.2.1dv.tgz 
 RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz
 
 RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
 #RUN sudo -u postgres createuser dvnapp
 
 # copy configuration related files
-RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ ; cp /tmp/dv/schema.xml /opt/solr-4.6.0/example/solr/collection1/conf/schema.xml
+RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
+RUN cp -r /opt/solr-7.2.1/server/solr/configsets/_default /opt/solr-7.2.1/server/solr/collection1
+RUN cp /tmp/dv/schema.xml /opt/solr-7.2.1/server/solr/collection1/conf/schema.xml
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping glassfish user and solr user (run both as root)
 

conf/docker-aio/entrypoint.bash

@@ -1,9 +1,12 @@
 #!/usr/bin/env bash
 
 sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
-cd /opt/solr-4.6.0/example/
-java -DSTOP.PORT=8079 -DSTOP.KEY=a09df7a0d -jar start.jar &
+cd /opt/solr-7.2.1/
+# TODO: Run Solr as non-root and remove "-force".
+bin/solr start -force
+bin/solr create_core -c collection1 -d server/solr/collection1/conf -force
 
+# TODO: Run Glassfish as non-root.
 cd /opt/glassfish4
 bin/asadmin start-domain
 sleep infinity

conf/docker-aio/readme.txt

@@ -2,12 +2,11 @@ first pass docker all-in-one image, intended for running integration tests again
 
 Could be potentially usable for normal development as well.
 
-
 Initial setup (aka - do once):
-- Do surgery on glassfish4 and solr4.6.0 following guides, place results in `conf/docker-aio/dv/deps` as `glassfish4dv.tgz` and `solr-4.6.0dv.tgz` respectively. If you `cd conf/docker-aio` and run `./0prep_deps.sh` these tarballs will be constructed for you.
+- `cd conf/docker-aio` and run `./0prep_deps.sh` to created Glassfish and Solr tarballs in `conf/docker-aio/dv/deps`.
 
 Per-build:
-- `cd conf/docker-aio`, and run `1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file
+- `cd conf/docker-aio`, and run `./1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file
 - build the docker image: `docker build -t dv0 -f c7.dockerfile .`
 
 - Run image: `docker run -d -p 8083:8080 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container)

conf/docker/build.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # Creates images and pushes them to Docker Hub.
-# The "kick-the-tires" tag should be relatively stable. No breaking changes.
+# The "latest" tag should be relatively stable. Don't push breaking changes there.
+# None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040
 # Push to custom tags or tags based on branch names to iterate on the images.
 if [ -z "$1" ]; then
   echo "No argument supplied. Please specify \"branch\" or \"custom my-custom-tag\" for experiments or \"stable\" if your change won't break anything."
@@ -31,11 +32,31 @@ echo Images will be pushed to Docker Hub with the tag \"$TAG\".
 docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf
 docker push iqss/dataverse-solr:$TAG
 # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip.
-# FIXME: Automate the building of dataverse.war and dvinstall.zip. Think about https://github.com/IQSS/dataverse/issues/3974 and https://github.com/IQSS/dataverse/pull/3975
+cd ../..
+mvn clean
+scripts/installer/custom-build-number
+mvn package
+cd conf/docker
 cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war
+if [[ "$?" -ne 0 ]]; then
+  echo "Unable to copy war file into place. Did 'mvn package' work?"
+  exit 1
+fi
+cd ../../scripts/installer
+make clean
+make
+cd ../../conf/docker
 cp ../../scripts/installer/dvinstall.zip dataverse-glassfish
-cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish
+if [[ "$?" -ne 0 ]]; then
+  echo "Unable to copy dvinstall.zip file into place. Did 'make' work?"
+  exit 1
+fi
 cp ../../downloads/glassfish-4.1.zip dataverse-glassfish
+if [[ "$?" -ne 0 ]]; then
+  echo "Unable to copy Glassfish zip file into place. You must run the download script in that directory once. "
+  exit 1
+fi
+# We'll assume at this point that the download script has been run.
 cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish
 docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish
 # FIXME: Check the output of `docker build` and only push on success.

conf/docker/dataverse-glassfish/.gitignore

@@ -2,4 +2,3 @@ glassfish-4.1.zip
 weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 dvinstall.zip
 dataverse.war
-default.config

conf/docker/dataverse-glassfish/default.config

@@ -0,0 +1,16 @@
+HOST_DNS_ADDRESS	localhost
+GLASSFISH_DIRECTORY	/usr/local/glassfish4
+ADMIN_EMAIL	 
+MAIL_SERVER	mail.hmdc.harvard.edu
+POSTGRES_ADMIN_PASSWORD	secret
+POSTGRES_SERVER	dataverse-postgresql-service
+POSTGRES_PORT	5432
+POSTGRES_DATABASE	dvndb
+POSTGRES_USER	dvnapp
+POSTGRES_PASSWORD	secret
+SOLR_LOCATION	dataverse-solr-service:8983
+TWORAVENS_LOCATION	NOT INSTALLED
+RSERVE_HOST	localhost
+RSERVE_PORT	6311
+RSERVE_USER	rserve
+RSERVE_PASSWORD	rserve

conf/docker/dataverse-glassfish/entrypoint.sh

@@ -126,6 +126,13 @@ if [ "$1" = 'dataverse' ]; then
 #        ./setup-irods.sh
 #    fi
 
+    # We do change the Solr server in Minishift/OpenShift, which is
+    # the primary target for all of the work under conf/docker.
+    echo -e "\n\nRestarting Dataverse in case Solr host was changed..."
+    /usr/local/glassfish4/glassfish/bin/asadmin stop-domain
+    sleep 3
+    /usr/local/glassfish4/glassfish/bin/asadmin start-domain
+
     echo -e "\n\nDataverse started"
 
     sleep infinity

conf/docker/solr/Dockerfile

@@ -3,21 +3,12 @@ MAINTAINER Dataverse (support@dataverse.org)
 
 RUN yum install -y wget unzip perl git java-1.8.0-openjdk-devel postgresql.x86_64
 
-# Install Solr 4.6.0
+# Install Solr 7.2.1
 # The context of the build is the "conf" directory.
-COPY solr/4.6.0/schema.xml /tmp
+COPY solr/7.2.1/schema.xml /tmp
+COPY solr/7.2.1/solrconfig.xml /tmp
 
-RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz  && \
-	tar xvzf solr-4.6.0.tgz  && \
-	mv solr-4.6.0 /usr/local/  && \
-	cd /usr/local/solr-4.6.0/example/solr/collection1/conf/  && \
-	mv schema.xml schema.xml.backup  && \
-	cp /tmp/schema.xml . && \
-	rm /tmp/solr-4.6.0.tgz
-
-RUN ln -s /usr/local/solr-4.6.0/example/logs /var/log/solr
-
-VOLUME  /usr/local/solr-4.6.0/example/solr/collection1/data
+RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz
 
 EXPOSE 8983
 

conf/docker/solr/entrypoint.sh

@@ -1,8 +1,19 @@
 #!/bin/bash
 
+# FIXME: Don't run Solr out of /tmp!
+# Solr is /tmp to avoid AccessDeniedException under Minishift/OpenShift.
+SOLR_DIR=/tmp/solr-7.2.1
+
 if [ "$1" = 'solr' ]; then
-    cd /usr/local/solr-4.6.0/example/
-    java -jar start.jar 
+    cd /tmp
+    tar xvfz solr-7.2.1.tgz
+    cp -r $SOLR_DIR/server/solr/configsets/_default $SOLR_DIR/server/solr/collection1
+    cp /tmp/schema.xml $SOLR_DIR/server/solr/collection1/conf
+    cp /tmp/solrconfig.xml $SOLR_DIR/server/solr/collection1/conf
+    cd $SOLR_DIR
+    bin/solr start
+    bin/solr create_core -c collection1 -d server/solr/collection1/conf
+    sleep infinity
 elif [ "$1" = 'usage' ]; then
     echo  'docker run -d iqss/dataverse-solr solr'
 else