diff --git a/config/core/adhoc/ansible.cfg b/config/core/adhoc/ansible.cfg new file mode 100644 index 000000000..e0c9ee54e --- /dev/null +++ b/config/core/adhoc/ansible.cfg @@ -0,0 +1,12 @@ +; This file is used when performing ad hoc commands, e.g. +; `ansible localhost -m git -a "repo=http://repo.com dest=/opt/abc version=dev"` +; Scripts (or humans) must `cd` into this directory so Ansible knows to use this +; file versus any other. + +[defaults] + +; makes output for ad hoc commands in json, so jq can process it +stdout_callback = json + +; use callbacks on ad hoc commands +bin_ansible_callbacks=True diff --git a/src/roles/base/templates/config.php.j2 b/src/roles/base/templates/config.php.j2 index c6a45ef15..ff63ec9af 100644 --- a/src/roles/base/templates/config.php.j2 +++ b/src/roles/base/templates/config.php.j2 @@ -1,6 +1,9 @@ > /opt/data-meza/logs/autodeploy-`date "+\%Y-\%m-\%d"`.log 2>&1 +# + +# Don't allow errors +set -e + +echo "Start meza auto-deployer" +echo $(date "+%Y-%m-%d %H:%M:%S") + +# Path to this file's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# must be root or sudoer +if [ "$(whoami)" != "root" ]; then + echo "Try running this script with sudo: \"sudo bash unite-the-wikis.sh\"" + exit 1 +fi + +# Check if a deploy is happening. +# +# This will cause the script to exit if a deploy is currently underway, thus +# preventing two deploys from happening at once. +# +# FIXME: not implemented yet +# source "$DIR/check-deploy.sh" + +# Gets info about public config +source /opt/.deploy-meza/config.sh + +# +# FIXME: For now, don't touch secret config. At some point find a way to +# configure it's repo and version. + +if [ -z "$local_config_repo_repo" ]; then + >&2 echo "Auto-deploy requires 'local_config_repo' set in secret or public config" + exit 1; +fi + +if [ -z "$enforce_meza_version" ]; then + >&2 echo "Auto-deploy requires 'enforce_meza_version' var set in public or secret config" + exit 1; +fi + +# Set Slack notify variables that are the same for all notifications +if [ ! -z "$autodeployer_slack_token" ]; then SLACK_TOKEN="$autodeployer_slack_token"; fi +if [ ! -z "$autodeployer_slack_username" ]; then SLACK_USERNAME="$autodeployer_slack_username"; fi +if [ ! -z "$autodeployer_slack_channel" ]; then SLACK_CHANNEL="$autodeployer_slack_channel"; fi +if [ ! -z "$autodeployer_slack_icon_url" ]; then SLACK_ICON_URL="$autodeployer_slack_icon_url"; fi + +# If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script +if [ ! -z "$SLACK_TOKEN" ]; then + NOTIFY="$DIR/slack-notify.sh" +else + NOTIFY="$DIR/no-notify.sh" +fi + +GIT_FETCH="$DIR/git-fetch.sh" + +# Set PUBLIC config version +# +# Could optionally set public config's repo in secret config, but since that is +# not done universally, not going to enforce it here. Just use whatever repo is +# currently being used as origin. +PUBLIC_CONFIG_DEST="/opt/conf-meza/public" +PUBLIC_CONFIG_REPO="$local_config_repo_repo" +PUBLIC_CONFIG_VERSION="$local_config_repo_version" +PUBLIC_CONFIG_CHANGE=$($GIT_FETCH "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_VERSION") + +# +# Check if attempt to git-pull PUBLIC CONFIG failed +# +# FIXME: For some reason the jq command below was not working if it was within +# the conditional, so it has to be out here, where it forces us to +# temporarily allow errors. +set +e +echo "Did git fetch fail on public config?" +echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e +if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$PUBLIC_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating public config failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; +fi + +# +# Check if changes were made to PUBLIC CONFIG +# +echo "Were there changes to public config?" +echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e +if [ $? -eq 0 ]; then + PUBLIC_CONFIG_BEFORE_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + PUBLIC_CONFIG_AFTER_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $PUBLIC_CONFIG_BEFORE_HASH" + echo "After hash: $PUBLIC_CONFIG_BEFORE_HASH" + + pushd "$PUBLIC_CONFIG_DEST" + PUBLIC_CONFIG_DIFF=$(git diff "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" 2>&1) + pushd +else + PUBLIC_CONFIG_DIFF="" + PUBLIC_CONFIG_AFTER_HASH="" +fi + +# Set MEZA version +MEZA_REPO="https://github.com/enterprisemediawiki/meza" +MEZA_DEST="/opt/meza" +MEZA_VERSION="$enforce_meza_version" +MEZA_CHANGE=$($GIT_FETCH "$MEZA_REPO" "$MEZA_DEST" "$MEZA_VERSION") + +# +# Check if attempt to git-pull MEZA failed +# +echo "Did git fetch fail on Meza?" +echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e +if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$MEZA_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating Meza failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; +fi + +# +# Check if changes were made to MEZA +# +echo "Were there changes to Meza?" +echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e +if [ $? -eq 0 ]; then + MEZA_BEFORE_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + MEZA_AFTER_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $MEZA_BEFORE_HASH" + echo "After hash: $MEZA_AFTER_HASH" +else + MEZA_AFTER_HASH="" +fi +set -e # end FIXME from above. + + +# +# Neither Meza mor config changed? Exit. +# +if [ -z "$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; then + echo "Nothing to deploy" + exit 0; +fi + +# +# Notify if PUBLIC CONFIG changed +# +if [ ! -z "$PUBLIC_CONFIG_AFTER_HASH" ]; then + + MESSAGE=$(cat <<-END + Public config changed versions: + FROM: \`$PUBLIC_CONFIG_BEFORE_HASH\` + TO: \`$PUBLIC_CONFIG_AFTER_HASH\` + + Tracking version: \`$PUBLIC_CONFIG_VERSION\` + + Diff: + \`\`\` + $PUBLIC_CONFIG_DIFF + \`\`\` +END +) + + echo -e "$MESSAGE" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$MESSAGE" + SLACK_COLOR="good" + source $NOTIFY + fi +fi + +# +# Notify if MEZA changed +# +if [ ! -z "$MEZA_AFTER_HASH" ]; then + + MESSAGE=$(cat <<-END + Meza application changed versions: + FROM: \`$MEZA_BEFORE_HASH\` + TO: \`$MEZA_AFTER_HASH\` + + Tracking version: \`$MEZA_VERSION\` +END +) + + echo -e "$MESSAGE" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$MESSAGE" + SLACK_COLOR="good" + source $NOTIFY + fi +fi + + +# +# Do deploy +# +echo "Deploying" +DEPLOY_TYPE="Deploy" +DEPLOY_ARGS="--tags base --skip-tags mediawiki" # autodeploy deploys everything ... but while testing keep it really light +DEPLOY_LOG_PREFIX="deploy-after-config-change-" +source "$DIR/do-deploy.sh" +echo "Done" diff --git a/src/scripts/autodeployer/do-deploy.sh b/src/scripts/autodeployer/do-deploy.sh new file mode 100755 index 000000000..61fc0647c --- /dev/null +++ b/src/scripts/autodeployer/do-deploy.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Do deploy. Notify on success. Notify and retry on fail. +# +# To use this script, either pass in DEPLOY_TYPE, DEPLOY_ARGS, AND LOG_PREFIX, +# or set them ahead of time and `source` the script: +# +# 1. ./do-deploy.sh "Backup" "--skip-tags search-index" "test-deploy-" +# +# 2. DEPLOY_TYPE="Backup" +# DEPLOY_ARGS="--skip-tags search-index" +# LOG_PREFIX="test-deploy-" +# +# Args: +# DEPLOY_TYPE: Just used in notifications for type of deploy, e.g. "Backup +# starting" or "Deploy starting" where "Backup" and "Deploy" are +# the DEPLOY_TYPE +# DEPLOY_ARGS: Any arguments that are going to get added to the deploy +# command. So if you want to do: +# `meza deploy dev --tags mediawiki --skip-tags latest` +# The DEPLOY_ARGS would be "--tags mediawiki --skip-tags latest" +# LOG_PREFIX: Logs are written to /opt/data-meza/logs to a file ending in +# the date/time and ".log". Prefix it with something like +# "nightly-backup-" to make "nightly-backup-$DATETIME.log" + +# Path to this file's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + + +# Setting configuration for DEPLOY_TYPE, DEPLOY_ARGS, and LOG_PREFIX +# +# If any of them is being set via script argument $1, $2, or $3, respectively, +# then use that value. Otherwise check if they already have a value set earlier +# within the enviroment, and set a default if not. +if [ ! -z "$1" ]; then + DEPLOY_TYPE="$1" +elif [ -z "$DEPLOY_TYPE" ]; then + DEPLOY_TYPE="Deploy" +fi + +if [ ! -z "$2" ]; then + DEPLOY_ARGS="$2" +elif [ -z "$DEPLOY_ARGS" ]; then + DEPLOY_ARGS="" +fi + +if [ ! -z "$3" ]; then + LOG_PREFIX="$3" +elif [ -z "$LOG_PREFIX" ]; then + LOG_PREFIX="deploy-" +fi + +# Gets info about public config +source /opt/.deploy-meza/config.sh + +# If SLACK_TOKEN not set from outside this script, grab from config.sh +if [ -z "$SLACK_TOKEN" ]; then + SLACK_TOKEN="$autodeployer_slack_token" +fi + +# If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script +if [ ! -z "$SLACK_TOKEN" ]; then + NOTIFY="$DIR/slack-notify.sh" +else + NOTIFY="$DIR/no-notify.sh" +fi + +source $NOTIFY "$DEPLOY_TYPE starting" "good" + +# First try at deploy. Allow failures so we can capture them later +set +e +meza deploy "$m_environment" $DEPLOY_ARGS \ + > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 + +# If deploy success, notify. Else retry once. +if [ $? -eq 0 ]; then + source $NOTIFY "$DEPLOY_TYPE complete" "good" +else + source $NOTIFY "$DEPLOY_TYPE attempt failed. Retrying..." "warning" + meza deploy "$m_environment" $DEPLOY_ARGS \ + > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 + + if [ $? -eq 0 ]; then + source $NOTIFY "$DEPLOY_TYPE complete" "good" + else + source $NOTIFY "$DEPLOY_TYPE failed" "danger" + fi +fi + + + + diff --git a/src/scripts/autodeployer/git-fetch.sh b/src/scripts/autodeployer/git-fetch.sh new file mode 100755 index 000000000..abbab7769 --- /dev/null +++ b/src/scripts/autodeployer/git-fetch.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Use Ansible's Git module plus jq to easily set repository versions + +# Get directory to return to at end +CWD=$(pwd) + +# change to directory holding appropriate ansible.cfg file for this operation +cd /opt/meza/config/core/adhoc + +REPO="$1" +DEST="$2" +VERSION="$3" + +ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" + +cd "$CWD" diff --git a/src/scripts/autodeployer/no-notify.sh b/src/scripts/autodeployer/no-notify.sh new file mode 100755 index 000000000..f73d123bc --- /dev/null +++ b/src/scripts/autodeployer/no-notify.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# +# Script used to handle when no notification system is set + +echo -e "$(date "+\%Y-\%m-\%d"): No notification method set.\n Color: $2\n Message: $1" >> /opt/data-meza/logs/missing-notifier.log diff --git a/src/scripts/autodeployer/slack-notify.sh b/src/scripts/autodeployer/slack-notify.sh new file mode 100755 index 000000000..fa4e14fad --- /dev/null +++ b/src/scripts/autodeployer/slack-notify.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# +# Use Ansible's Slack module to send messages. +# +# Required variables +# SLACK_TOKEN="" # Exits script if you don't set this. +# +# Recommended variables +# SLACK_MESSAGE="" # Defaults to a pointless message. Set this one. +# SLACK_COLOR="" # Defaults to "good" which means "green". Also available +# are "warning" (orange) and "danger" (red) or using hex +# code (e.g. #439FE0 = light blue) +# +# Optional variables +# SLACK_USERNAME="" # Will use your Slack token's default username by default +# SLACK_CHANNEL="" # Will use your Slack token's default channel by default +# SLACK_ICON_URL="" # will use Meza logo by default + +# Get directory to return to at end +CWD=$(pwd) + +# change to directory holding appropriate ansible.cfg file for this operation +cd /opt/meza/config/core/adhoc + + + +if [ -z "$SLACK_TOKEN" ]; then + >&2 echo "You need to set a SLACK_TOKEN variable" + exit 1; +fi + +# if first param not empty, use it for SLACK_MESSAGE +if [ ! -z "$1" ]; then + SLACK_MESSAGE="$1" +elif [ -z "$SLACK_MESSAGE" ]; then + SLACK_MESSAGE="Empty message." +fi + +# if second param not empty, use it for SLACK_COLOR +if [ ! -z "$2" ]; then + SLACK_COLOR="$2" +elif [ -z "$SLACK_COLOR" ]; then + SLACK_COLOR="good" # assume all is well +fi + +if [ -z "$SLACK_CHANNEL" ]; then + SLACK_CHANNEL_WITH_PARAM="channel='$SLACK_CHANNEL'" +else + SLACK_CHANNEL_WITH_PARAM="" # use default for token +fi + +if [ -z "$SLACK_USERNAME" ]; then + SLACK_USERNAME_WITH_PARAM="username='$SLACK_USERNAME'" +else + SLACK_USERNAME_WITH_PARAM="" # use default for token +fi + +if [ -z "$SLACK_ICON_URL" ]; then + SLACK_ICON_URL="https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png" +fi + + +# Escape single quotes +SLACK_MESSAGE=$(echo "$SLACK_MESSAGE" | sed "s/'/\\\'/g") + +# Turn on allowing failures +set +e + +# for debug +echo "DEBUG OUTPUT OF SLACK NOTIFY COMMAND" +echo \ +"token='$SLACK_TOKEN' \ +$SLACK_CHANNEL_WITH_PARAM \ +msg='$SLACK_MESSAGE' \ +$SLACK_USERNAME_WITH_PARAM \ +icon_url=$SLACK_ICON_URL \ +link_names=1 \ +color=$SLACK_COLOR" + +# Attempt to send message +ansible localhost -m slack -a \ +"token='$SLACK_TOKEN' \ +$SLACK_CHANNEL_WITH_PARAM \ +msg='$SLACK_MESSAGE' \ +$SLACK_USERNAME_WITH_PARAM \ +icon_url=$SLACK_ICON_URL \ +link_names=1 \ +color=$SLACK_COLOR" + +# If message fails, send a generic message +if [ $? -eq 0 ]; then + echo "Slack notify success" +else + echo "Slack notify fail. Attempted message was:" + echo "$SLACK_MESSAGE" + SLACK_MESSAGE="Slack message failed. See logs for attempted message." + ansible localhost -m slack -a \ +"token='$SLACK_TOKEN' \ +$SLACK_CHANNEL_WITH_PARAM \ +msg='$SLACK_MESSAGE' \ +$SLACK_USERNAME_WITH_PARAM \ +icon_url=$SLACK_ICON_URL \ +link_names=1 \ +color=$SLACK_COLOR" +fi + +# Turn off allowing errors +set -e + +cd "$CWD"