adpn-ingest-test

#!/bin/bash
#
# adpn-ingest-test: Assemble information and run tests for a package of files staged for
# ingest as a LOCKSS Archival Unit, given a local location, a LOCKSS Plugin and parameters
# for the plugin.
#
# Switches can be provided on the command line, or interactively on the console at input
# prompts. You can set default values for commonly-repeated switches (for example, if your
# LOCKSS daemon is always at the same location, you won't need to re-enter that location
# every time you run the script) by creating a plaintext file called:
#
# 		./adpn-ingest-test.defaults.conf
#
# where the conf file is located in the same directory as the adpn-ingest-test script.
# The file should be a text/plain file, with one switch on each line of the file. If the
# same switch is provided on the command line, the value on the command line overrides the
# value provided in the defaults.conf file.
#
# @version 2021.0924

__doc__="""
Usage:

  Interactive:
  
  adpn-ingest-test --local=<PATH>

    run without other parameters, the script will prompt on the console for AU title,
    publisher plugin, and LOCKSS Daemon information, report back the required
    parameters for the Publisher Plugin, and then allow the option to input parameters
    and perform a test of the accessibility of the start URL and/or manifest page for
    an HTTP GET.
	
  Scripted:
  
  adpn-ingest-test --local=<PATH> [OPTION]...

    --local=<PATH>            	path to the files being put into the AU (e.g.: 'w:\\Path')
    --au_title=<TITLE>        	title for this AU in titledb (e.g.: 'ADAH Lovely Files')
    --<PARAM>=<VALUE>         	set plugin parameter <PARAM> to <VALUE>
                              	  (e.g.: subdirectory=Lovely-Files)

  To retrieve file size and file count information from an FTP staging server:
    --staged=<URL>            	FTP location for base directory or exact location of
                              	staged AU files (e.g.: ftp://user@server/Lockss/)
    --subdirectory=<DIR>      	optional, subdirectory within the FTP base directory that
                              	  contains the staged AU files. If provided, it will also
                              	  be passed along as a parameter to the publishing plugin.

  To select and configure a plugin for this ingest, use:
    --jar=<URL>               	URL directly to a plugin's JAR file on the props server
    --plugin=<NAME>           	use the plugin named <NAME>
    --plugin-id=<FQCN>        	use the plugin with uniqid <FQCN>
    --plugin-regex=<PATTERN>  	use the plugin whose name matches <PATTERN>
    --plugin-keywords=<WORDS> 	use the plugin whose name contains keywords <WORDS>

  To connect to your LOCKSS daemon (to retrieve plugin lists):  
    --daemon=<HOST>:<PORT>    	Your LOCKSS daemon host (e.g.: adpnadah.alabama.gov:8081)
    --user=<USERNAME>         	The HTTP Auth username for your LOCKSS Daemon
    --pass=<PASSWORD>         	The HTTP Auth password for your LOCKSS Daemon
  
  Plugins will be hosted by the LOCKSS props server/admin node. If you need to connect
  to the LOCKSS props server through a SOCKS5 proxy, use:
    --proxy=<HOST>            	the name of your proxy (use localhost for an SSH tunnel)
    --port=<NUMBER>           	the port number for your proxy
  
  If you need to use SSH tunneling to connect to the SOCKS5 proxy, use:
    --tunnel=<HOST>          	the name of the host to open an SSH tunnel to
    --tunnel-port=<NUMBER>   	the port for SSH connections to the tunnel (default: 22)

  Network Admin:
    
  <input> | adpn-ingest-test - --remote [OPTION]...
  adpn-ingest-test <JSONFILE> --remote [OPTION]...
  
  JSON input provided from the local side can provide the au_title, plugin selection and
  values for required plugin parameters. Parameters for connecting to the LOCKSS daemon
  and to the props server are same as above. If you need to override a value provided by
  the JSON input, you can do so with a command-line switch (e.g., a stupid title provided
  in JSON can be overridden with --au_title='ADAH Better Title').
"""

##########################################################################################
### CONFIGURATION: Set up some constants #################################################
##########################################################################################

SCRIPTPATH=$(which "$0")
SCRIPTPATH=$(readlink -f "${SCRIPTPATH}")
SCRIPTDIR=$(dirname "${SCRIPTPATH}")
SCRIPT=$(basename "${SCRIPTPATH}")

source "${SCRIPTDIR}/adpn-define-aliases"

declare -A SUBARGV_LOCKSSPLUGINURL ; SUBARGV_LOCKSSPLUGINURL=()
declare -A SUBARGV_ADPNPLUGININFO ; SUBARGV_ADPNPLUGININFO=()
declare -A SUBARGV_LOCKSSRETRIEVEJAR ; SUBARGV_LOCKSSRETRIEVEJAR=()
declare -A SUBARGV_TESTURLOK ; SUBARGV_TESTURLOK=()

##########################################################################################
### DEPENDENCIES: check for required command-line tools and Python scripts ###############
##########################################################################################

declare -A DEPENDENCIES ; DEPENDENCIES=(
	[mktemp]="command-line %s tool"
	[grep]="command-line %s tool"
	[xargs]="command-line %s tool"
	[du]="command-line %s tool"
	[find]="command-line %s tool"
	[adpn-plugin-info]="%s Bash script"
	[lockss-plugin-url.py]="%s Python script"
	[lockss-retrieve-jar.py]="%s Python script"
	[lockss-plugin-props.py]="%s Python script"
	[lockss-plugin-props-print-parameter.py]="%s Python script"
	[adpn-ingest-test-url-ok.py]="%s Python script"
)

DEPENDENCY_FAILURE=""
for _CMD in "${!DEPENDENCIES[@]}" ; do
	DESC=$(printf "${DEPENDENCIES[$_CMD]}" "${_CMD}")
	if [[ -z $(which ${_CMD}) ]] ; then
		adpn_notice "${LINENO}" "Dependency Failure: %s. This script requires the %s" "${_CMD}" "${DESC}". 1>&2
		DEPENDENCY_FAILURE="${DEPENDENCY_FAILURE}+${_CMD}"
	fi
done

if [[ ! -z "${DEPENDENCY_FAILURE}" ]] ; then
	exit 255
fi

##########################################################################################
### COMMAND LINE: loop through switches and sort them to subsidiary scripts ##############
##########################################################################################

JAR=""
HELP=""

declare -a SWITCHFILES ; SWITCHFILES=()
[[ -r "${CONFFILE}" ]] && SWITCHFILES+=(${CONFFILE})

CMDLINETXT=$(mktemp)
JSONSWITCHES=$(mktemp)
declare -a _CMDLINE=("$0")
until [[ "$#" -lt 1 ]] ; do
	_CMDLINE+=( "$1" )
	printf "%s\n" "$1" >> "${CMDLINETXT}"
	shift
done

SWITCHFILES+=(${CMDLINETXT})

adpn_command_line "${SWITCHFILES[@]}"
adpn_set_display_settings # V, VV, Q, QQ, DBG, DBGLEVEL, DDBG, SCRIPT_CMD_NAME / @see adpn-define-aliases

adpn_debug 10,"${SCRIPT}" "${LINENO}" "command line: %s" "$( printf "%q " "${_CMDLINE[@]}" )"

if [[ "${_ARGV[1]}" == "-" || -r "${_ARGV[1]}" ]] ; then
	if [[ "${DBGLEVEL}" -ge 1 ]] ; then
		printf "%s" "[${SCRIPT}] reading JSON input from: ${_ARGV[1]}" 1>&2
		if [[ "${DBGLEVEL}" -ge 2 ]] ; then
			printf " <<<EOF\n%sEOF;" "$( cat "${_ARGV[1]}" )" 1>&2
		fi
		printf "\n" 1>&2
	fi
	
	adpn-json-to-switches.py "${_ARGV[1]}" >> "${JSONSWITCHES}" ; AJTS_ERRCODE="$?"
	if [[ "${AJTS_ERRCODE}" -eq 255 ]] ; then
		echo "[${SCRIPT}] Aborted by user keyboard break (Ctrl-C)." 1>&2
		exit 255
	fi
fi

for DEFAULT in "daemon" "daemon/user" "daemon/pass" ; do
	DEFAULT_VALUE=$(adpnprop "${DEFAULT}")
	if [[ -n "${DEFAULT_VALUE}" ]] ; then
		DEFAULT_KEY="$( basename "${DEFAULT}" )"
		printf -- "--%s=%s\n" "${DEFAULT_KEY}" "${DEFAULT_VALUE}" >> "${JSONSWITCHES}"
	fi 
done

SWITCHFILES=()
[[ -r "${CONFFILE}" ]] && SWITCHFILES+=( "${CONFFILE}" )
[[ -r "${JSONSWITCHES}" ]] && SWITCHFILES+=( "${JSONSWITCHES}" )
[[ -r "${CMDLINETXT}" ]] && SWITCHFILES+=( "${CMDLINETXT}" )
ADPN_INGEST_TEST_DISPLAY=ingest

adpn_command_line "${SWITCHFILES[@]}"

for KEY in "${!_PARAM[@]}" ; do
	VALUE="${_PARAM[$KEY]}"
	SWITCH_KEY="$( printf "${KEY}" | sed -E 's/[#][0-9]+$//g' )"
	case "${SWITCH_KEY}" in
		"jar")
			JAR="${VALUE}"
			;;
		"help")
			ADPN_INGEST_TEST_DISPLAY="${KEY}"
			;;
		"batch"|"local"|"remote"|"verbose")
			# NOOP: adpn-ingest-test operational parameters, not a passthru
			;;
		"au_title"|"staged"|"staged-to"|"staged-by"|"verified-by"|"packaged-in"|"packaged-by"|"gitlab-issue"|"gitlab-resource"|"file-size"|"peer"|"peer-from"|"peer-to"|"ingest-report"|"gitlab-issue"|"gitlab-resource"|"adpn:workflow")
			# NOOP: adpn-ingest-test descriptive passthru parameters
			;;
		"daemon"|"plugin"|"plugin-regex"|"plugin-keywords"|"plugin-id"|"user")
			SUBARGV_LOCKSSPLUGINURL[$KEY]="--${KEY}=${VALUE}"
			;;
		"pass")
			if [[ -n "${VALUE}" ]] ; then
				VALUE="$( adpn_getpassword_from "${VALUE}" )"
			fi
			SUBARGV_LOCKSSPLUGINURL["${KEY}"]="--${KEY}=${VALUE}"
			;;
		"proxy"|"port"|"tunnel"|"tunnel-port")
			SUBARGV_LOCKSSRETRIEVEJAR[$KEY]="--${KEY}=${VALUE}"
			SUBARGV_TESTURLOK[$KEY]="--${KEY}=${VALUE}"
			;;
		"subdirectory"|"directory")
			SUBARGV_ADPNPLUGININFO[$KEY]="--${KEY}=${VALUE}"
			;;
		*)
			SUBARGV_ADPNPLUGININFO[$KEY]="--${KEY}=${VALUE}"
			;;
	esac
done

adpn_debug 15,"${SCRIPT}" "${LINENO}" "parameters: %s %s" "$(declare -p _ARGV)" "$(declare -p _PARAM)"

rm "${JSONSWITCHES}"
rm "${CMDLINETXT}"

if [[ "${ADPN_INGEST_TEST_DISPLAY}" == "help" ]] ; then
	echo "${__doc__}"
	exit
fi

##########################################################################################
### USER INPUT: get missing parameters from the user console if not command line #########
##########################################################################################

if [[ -z "${_PARAM[au_title]}" && -z "${_PARAM[batch]}" ]] ; then
	read -r -p "Ingest Name/Title: " AU_TITLE
	_ARGV+=("--au_title=${AU_TITLE}")
	_PARAM[au_title]="${AU_TITLE}"
fi

if [[ -z "${_PARAM[local]}" && -z "${_PARAM[remote]}" && -z "${_PARAM[batch]}" ]] ; then
	read -r -p "Local file location: " LOCALPATH
	_ARGV+=("--local=${LOCALPATH}")
	_PARAM[local]="${LOCALPATH}"
fi

if [[
	-z "${JAR}"
	&& -z "${SUBARGV_LOCKSSPLUGINURL[plugin]}" 
	&& -z "${SUBARGV_LOCKSSPLUGINURL[plugin-regex]}" 
	&& -z "${SUBARGV_LOCKSSPLUGINURL[plugin-keywords]}" 
	&& -z "${SUBARGV_LOCKSSPLUGINURL[plugin-id]}" 
	&& -z "${_PARAM[batch]}"
]] ; then
	read -r -p "Publisher Plugin (keywords): " PUBLISHERPLUGIN
	_ARGV+=("--plugin-keywords=${PUBLISHERPLUGIN}")
	SUBARGV_LOCKSSPLUGINURL[plugin-keywords]="--plugin-keywords=${PUBLISHERPLUGIN}"
fi

if [[ -z "${SUBARGV_LOCKSSPLUGINURL[daemon]}" ]] ; then
	read -r -p "LOCKSS Daemon Server: " DAEMON
	_ARGV+=("--daemon=${DAEMON}")
	SUBARGV_LOCKSSPLUGINURL[daemon]="--daemon=${DAEMON}"
fi

if [[ -n "${_PARAM[daemon]}" && "-" != "${_PARAM[daemon]}" ]] ; then
	if [[ -z "${SUBARGV_LOCKSSPLUGINURL[user]}" ]] ; then
		read -r -p "LOCKSS Daemon Username: " DAEMONUSER
		_ARGV+=("--user=${DAEMONUSER}")
		SUBARGV_LOCKSSPLUGINURL[user]="--user=${DAEMONUSER}"
	fi
	
	if [[ -z "${SUBARGV_LOCKSSPLUGINURL[pass]}" ]] ; then
		read -r -s -p "LOCKSS Daemon Password: " DAEMONPASS ; echo
		_ARGV+=("--pass=${DAEMONPASS}")
		SUBARGV_LOCKSSPLUGINURL[pass]="--pass=${DAEMONPASS}"
	fi
fi

##########################################################################################
### lockss-plugin-url.py: Retrieve URL of the Plugin JAR package. ########################
##########################################################################################

PLUGINTXT=$(mktemp)

NONUNIQUE_URL=0
if [[ -z "${JAR}" ]] ; then
	JAR=$( lockss-plugin-url.py "${SUBARGV_LOCKSSPLUGINURL[@]}" )
	NONUNIQUE_URL="$?"
fi

if [[ -z "${JAR}" ]] ; then

	EXITCODE=2
	echo "[${SCRIPT}] No Publisher Plugin matches ${SUBARGV_LOCKSSPLUGINURL[@]}." 1>&2

elif [[ "${NONUNIQUE_URL}" -eq 2 ]] ; then

	EXITCODE=2
	echo_header "Multiple Matching Plugins:" "="
	echo "${JAR}"
	
else

	EXITCODE=0
	
	FILESIZE_STR=""
	if [ ! -z "${_PARAM[remote]}" ] ; then
		
		if [ ! -z "${_PARAM[peer]}" ] ; then
			_PARAM[peer-to]="${_PARAM[peer]}"
		fi

		if [ ! -z "${_PARAM[file-size]}" ] ; then
			FILESIZE_STR="${_PARAM[file-size]}"
		fi

	elif [ ! -z "${_PARAM[staged]}" ] ; then

		if [ ! -z "${_PARAM[peer]}" ] ; then
			_PARAM[peer-from]="${_PARAM[peer]}"
		fi
		
		STAGED_CONTENT_FILE_SIZE=$(
			staged-content-file-size.py \
			--output=text/tab-separated-values \
			"${_PARAM[staged]}" \
			"${_PARAM[subdirectory]}"
		)
		DU_HUMANREADABLE=$(echo "${STAGED_CONTENT_FILE_SIZE}" | cut -f1)
		DU_BYTES=$(echo "${STAGED_CONTENT_FILE_SIZE}" | cut -f2)
		FIND_FILECOUNT=$(echo "${STAGED_CONTENT_FILE_SIZE}" | cut -f3)
		
	elif [ ! -z "${_PARAM[local]}" ] ; then

		if [ ! -z "${_PARAM[peer]}" ] ; then
			_PARAM[peer-from]="${_PARAM[peer]}"
		fi

		DU_BYTES=`du --human-readable --apparent-size --bytes "${_PARAM[local]}" | cut -f1`
		DU_BYTES=$(printf "%'d" "${DU_BYTES}")

		DU_HUMANREADABLE=`du --human-readable --apparent-size --human-readable --max-depth=0 "${_PARAM[local]}" | cut -f1`

		FIND_FILECOUNT=$(find "${_PARAM[local]}" | wc -l)

	fi

	if [[ ! -z "${DU_HUMANREADABLE}" ]] ; then
		FILESIZE_STR=$(printf "%s (%s bytes, %d files)\n" "${DU_HUMANREADABLE}" "${DU_BYTES}" "${FIND_FILECOUNT}")
	fi
	
	if [[ ! -z "${_PARAM[au_title]}" ]] ; then
		printf "Ingest Title\t%s\n" "${_PARAM[au_title]}" >> ${PLUGINTXT}
	fi
	if [[ ! -z "${FILESIZE_STR}" ]] ; then
		printf "File Size\t%s\n" "${FILESIZE_STR}" >> ${PLUGINTXT}
	fi

	declare -a TO_TSV=( \
		"peer-from" "peer-to" \
		"ingest-report" "adpn:workflow" \
		"staged-to" "staged-by" \
		"verified-by" \
		"packaged-in" "packaged-by" \
		"gitlab-issue" "gitlab-resource" \
	)
	for KEY in "${TO_TSV[@]}" ; do
		[[ -n "${_SWITCHES[$KEY]}" ]] && adpn-json-to-switches.py --reverse "${_SWITCHES[$KEY]}" >> "${PLUGINTXT}"
		
		I=2 ; KEY_I="${KEY}#${I}"
		while [[ -n "${_SWITCHES[$KEY_I]}" ]] ; do
			adpn-json-to-switches.py --reverse "${_SWITCHES[$KEY_I]}" >> "${PLUGINTXT}"
			I=$(( I + 1 )) ; KEY_I="${KEY}#${I}"
		done
	done
	
	if [[ ! -z "${_PARAM[pipeline]}" ]] ; then
		INGEST_STEP=$( printf "%s" "${_PARAM[pipeline]}" | cut --field=2 --delimiter=":" )
		printf "Ingest Step\t%s\n" "${INGEST_STEP}" >> ${PLUGINTXT}
	
		declare -A PIPE_STEP_NAMES=(
			[staged]="Staged"
			[ingested]="Accepted"
			[published]="Promoted"
		)
		if [[ -n "${_PARAM[by]}" ]] ; then
			KEY=$( strtotitle "${INGEST_STEP}" )
			[[ -n "${PIPE_STEP_NAMES[$INGEST_STEP]}" ]] && KEY="${PIPE_STEP_NAMES[$INGEST_STEP]}"
			printf "%s\t%s\n" "${KEY} By" "${_PARAM[by]}" >> "${PLUGINTXT}"
		fi
	
	fi
	
	######################################################################################
	### lockss-plugin-url.py | adpn-plugin-info: Retrieve & analyze Plugin JAR. ##########
	######################################################################################

	lockss-retrieve-jar.py --url="${JAR}" "${SUBARGV_LOCKSSRETRIEVEJAR[@]}" | adpn-plugin-info - --quiet --format=text/tab-separated-values --jar="${JAR}" "${SUBARGV_ADPNPLUGININFO[@]}" >> ${PLUGINTXT}
	LOCKSSRETRIEVEJAR="$?"
	
	if [[ "${LOCKSSRETRIEVEJAR}" -eq 0 ]] ; then
		echo_header "Ingest Information and Parameters:" "-"
	else
		echo_header "Ingest Information:" "-"
	fi

	pluginprop "${PLUGINTXT}" "Ingest Title"
	pluginprop "${PLUGINTXT}" "File Size"
	pluginprop "${PLUGINTXT}" "Plugin (JAR|Name|Version|ID)"
	pluginprop "${PLUGINTXT}" "Ingest (Report|Step)"
	
	if [[ "${LOCKSSRETRIEVEJAR}" -eq 0 ]] ; then
		pluginprop "${PLUGINTXT}" "PARAM[(].*[)]:"
	fi
	
	######################################################################################
	### adpn-ingest-test-url-ok.py: if all Plugin Parameters provided, test URLs OK ######
	######################################################################################

	if [[ "${LOCKSSRETRIEVEJAR}" -eq 0 ]] ; then
		echo ""
		echo -n "JSON PACKET:	"
		cat ${PLUGINTXT} | lockss-plugin-props-print-parameter.py --output=application/json
		
		echo_header "URL Retrieval Tests:" "-"
		( cat ${PLUGINTXT} | grep -P '^au_start_url\t' | adpn-ingest-test-url-ok.py "${SUBARGV_TESTURLOK[@]}" ) || ERROR_ON="${ERROR_ON} au_start_url"
		( cat ${PLUGINTXT} | grep -P '^au_manifest\t' | adpn-ingest-test-url-ok.py "${SUBARGV_TESTURLOK[@]}" ) || ERROR_ON="${ERROR_ON} au_manifest"		
		
	else
	
		EXITCODE=2		
	
		echo_header "Required Parameters:" "-"
		grep -P "^PARAM[(].*[)]:\t" ${PLUGINTXT} | lockss-plugin-props-print-parameter.py
		
		echo ""
		if [[ ! -z "${_PARAM[batch]}" ]] ; then
			read -n1 -p "Read from console (Y/N)? " PARAMS_YN ; echo ""
		else
			PARAMS_YN=N
		fi
		
		if [[ "${PARAMS_YN}" == "Y" || "${PARAMS_YN}" == "y" ]] ; then
			
			echo ""
			
			declare -a PLUGIN_LINE
			
			PARAMS_IN=$(mktemp)
			PARAMS_SWITCHES=$(mktemp)
			
			for idx in ${!_ARGV[@]} ; do
				if (( "${idx}" > 0 )) ; then
					printf "%s\n" "${_ARGV[$idx]}" >> ${PARAMS_SWITCHES}
				fi
			done
			
			grep -P "^PARAM[(].*[)]:\t" ${PLUGINTXT} > ${PARAMS_IN}
			while read -r -u3 -a PLUGIN_LINE ; do
				if [[ ! "${PLUGIN_LINE[1]}" =~ .*=.* ]] ; then
					read -p "${PLUGIN_LINE[0]} ${PLUGIN_LINE[1]}: " PARAM_VALUE
					echo "--${PLUGIN_LINE[1]}=${PARAM_VALUE}" >> "${PARAMS_SWITCHES}"
				fi
			done 3<"${PARAMS_IN}"
			
			xargs -a "${PARAMS_SWITCHES}" --delimiter='\n' "${_ARGV[0]}"
			EXITCODE="$?"
			
			rm "${PARAMS_IN}" "${PARAMS_SWITCHES}"
		fi
	fi
fi

##########################################################################################
### ERRORS: List any errors reported by subsidiary scripts ###############################
##########################################################################################

if [ ! -z "${ERROR_ON}" ] ; then
	EXITCODE=1
	
	echo_header "Ingest Test Errors:" "-"
	for mod in ${ERROR_ON} ; do
		echo "Failed: ${mod}"
	done
fi

##########################################################################################
### CLEANUP: remove temporary output file ################################################
##########################################################################################

rm ${PLUGINTXT}

exit ${EXITCODE}