From 758e7252d06d229385274c890006c0a590a99bb2 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Wed, 3 Jun 2020 16:35:18 +0800 Subject: [PATCH] chore: prepare Realtime for Marketplaces (#49) * chore: update Packer & Ansible files Update Packer & Ansible settings to prepare for AWS/DO Marketplaces * chore: set aws.json's `source_ami` to be user var `source_ami`'s default is set to x64 Ubuntu 18.04 EBS HVM in ap-southeast-1 (Singpore), but ami codes change between regions, so this should be a user variable along with `region`. * fix: get SECRET_KEY_BASE from env * chore: proper cleanup of Ansible tmp files * docs: better error messages for SECRET_KEY_BASE * ci: fix artifact naming of release assets --- .github/workflows/release.yml | 14 +- ansible/files/apt_periodic | 4 + ansible/files/realtime.env | 6 +- ansible/files/realtime.service.j2 | 4 +- ansible/playbook.yml | 12 +- ansible/tasks/setup-elixir.yml | 48 --- ansible/tasks/setup-realtime.yml | 50 +-- ansible/tasks/setup-system.yml | 46 +- aws.json | 34 +- do.json | 20 +- scripts/01-test | 9 + scripts/02-credentials_cleanup.sh | 1 + scripts/90-cleanup.sh | 36 ++ scripts/91-log_cleanup.sh | 5 + scripts/99-img_check.sh | 676 ++++++++++++++++++++++++++++++ server/config/config.exs | 1 - server/config/releases.exs | 11 +- 17 files changed, 860 insertions(+), 117 deletions(-) create mode 100644 ansible/files/apt_periodic delete mode 100644 ansible/tasks/setup-elixir.yml create mode 100644 scripts/01-test create mode 100644 scripts/02-credentials_cleanup.sh create mode 100644 scripts/90-cleanup.sh create mode 100644 scripts/91-log_cleanup.sh create mode 100755 scripts/99-img_check.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d993ede15..50a8d5fb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - '*' + - "[0-9]+.[0-9]+.[0-9]+*" jobs: release: @@ -22,12 +22,16 @@ jobs: elixir-version: 1.10.x otp-version: 22.x + - name: Get the version + id: get_version + run: echo ::set-output name=version::${GITHUB_REF#refs/tags/} + - name: Prepare release run: | mix deps.get mix compile mix release - tar -czvf realtime-ubuntu-latest.tar.gz -C ./_build/prod/rel/realtime/bin realtime + tar -czf realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz -C ./_build/prod/rel realtime env: MIX_ENV: prod @@ -39,8 +43,6 @@ jobs: with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} - draft: false - prerelease: false - name: Upload release assets id: upload-release-asset @@ -49,8 +51,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./server/realtime-ubuntu-latest.tar.gz - asset_name: realtime-ubuntu-latest.tar.gz + asset_path: ./server/realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz + asset_name: realtime-${{ steps.get_version.outputs.version }}-x86_64-linux-gnu.tar.gz asset_content_type: application/gzip - name: Upload build to Docker Hub diff --git a/ansible/files/apt_periodic b/ansible/files/apt_periodic new file mode 100644 index 000000000..5d37e9fc0 --- /dev/null +++ b/ansible/files/apt_periodic @@ -0,0 +1,4 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Download-Upgradeable-Packages "1"; +APT::Periodic::AutocleanInterval "7"; +APT::Periodic::Unattended-Upgrade "1"; diff --git a/ansible/files/realtime.env b/ansible/files/realtime.env index 0373a206f..a41bf2436 100644 --- a/ansible/files/realtime.env +++ b/ansible/files/realtime.env @@ -1,8 +1,8 @@ HOSTNAME=0.0.0.0 PORT=4000 DB_USER=postgres -DB_HOST=localhost +DB_HOST= DB_PORT=5432 DB_NAME=postgres -DB_PASSWORD=postgres -SECRET_KEY_BASE=SOMETHING_SUPER_SECRET +DB_PASSWORD= +SECRET_KEY_BASE= diff --git a/ansible/files/realtime.service.j2 b/ansible/files/realtime.service.j2 index f4b32f0bf..a279aa9c5 100644 --- a/ansible/files/realtime.service.j2 +++ b/ansible/files/realtime.service.j2 @@ -3,13 +3,13 @@ Description=Supabase Realtime server [Service] Type=simple -ExecStart=/opt/realtime/server/_build/prod/rel/realtime/bin/realtime start +ExecStart=/opt/realtime/bin/realtime start Restart=always RestartSec=3 # User for the build, and service User=realtime -EnvironmentFile=/etc/realtime.env +EnvironmentFile=/etc/realtime/realtime.env # Not specified in the supabase server docs but startup will fail if the HOME environmental # variable is not set. diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 35df57261..653587386 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -3,15 +3,13 @@ become: true vars: - supabase_commit: 951ef2350465d42eb6f741f4659ed5b6fda4cd7b - supabase_commit_checksum: sha1:eaf75edba248db39c881dc0dce5b129296afaecb - - erlang_solutions_deb: erlang-solutions_2.0_all.deb - erlang_solutions_deb_checksum: sha1:1968ec2ae81a5e1f56d2f173144926ec90a5e7c7 + realtime_version: 0.7.4 + realtime_checksum: sha1:9971212a8d39ada4385b97b44486e30230223116 tasks: - include_tasks: tasks/setup-system.yml - - include_tasks: tasks/setup-elixir.yml - - include_tasks: tasks/setup-realtime.yml + + - name: Remove temp dir at $HOME + shell: rm -rf ~/.ansible diff --git a/ansible/tasks/setup-elixir.yml b/ansible/tasks/setup-elixir.yml deleted file mode 100644 index 0d561650e..000000000 --- a/ansible/tasks/setup-elixir.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: System user - user: name=realtime - -# We use bsdtar instead of unzip, because we want to remove the first path component. -# E.g. when downloading a branch archive that first directory will be realtime-BRANCHNAME -# but when specifying a commit id the directory will be realtime-COMMITID -# bsdtar supports this feature, see http://archive.vn/M7DL8 -- name: System dependencies - apt: - pkg: - - bsdtar - -- name: Download commit archive - get_url: - url: "https://github.com/supabase/realtime/archive/{{ supabase_commit }}.zip" - dest: /tmp/realtime.zip - checksum: "{{ supabase_commit_checksum }}" - -- name: Create /opt/realtime - file: - path: /opt/realtime - state: directory - owner: realtime - -- name: Unpack archive to /opt/realtime - shell: bsdtar --strip-components=1 -xvf /tmp/realtime.zip - become_user: realtime - args: - chdir: /opt/realtime - -# Supabase server is build using Elixir and it uses the Elixir mix build tool -# See http://archive.vn/EqPby -- name: Download erlang-solutions deb - get_url: - url: "https://packages.erlang-solutions.com/{{ erlang_solutions_deb }}" - dest: /tmp/erlang-solutions.deb - checksum: "{{ erlang_solutions_deb_checksum }}" - -- name: Install erlang-solutions deb - apt: deb=file:///tmp/erlang-solutions.deb - -- name: Install esl-erlang and elixir - apt: - update_cache: yes - pkg: - - esl-erlang - - elixir diff --git a/ansible/tasks/setup-realtime.yml b/ansible/tasks/setup-realtime.yml index d57a69979..f7751e194 100644 --- a/ansible/tasks/setup-realtime.yml +++ b/ansible/tasks/setup-realtime.yml @@ -2,38 +2,29 @@ - name: System user user: name=realtime -- name: Install hex dependecy - shell: mix local.hex --force - become_user: realtime - args: - chdir: /opt/realtime/server - -- name: Install rebar dependency - shell: mix local.rebar --force - become_user: realtime - args: - chdir: /opt/realtime/server - -- name: Install other dependencies - shell: mix deps.get - become_user: realtime - args: - chdir: /opt/realtime/server +- name: Download release + get_url: + url: "https://github.com/supabase/realtime/releases/download/{{ realtime_version }}/realtime-{{ realtime_version }}-x86_64-linux-gnu.tar.gz" + dest: /tmp/realtime.tar.gz + checksum: "{{ realtime_checksum }}" + +- name: Unpack archive to /opt/realtime + unarchive: + remote_src: yes + src: /tmp/realtime.tar.gz + dest: /opt + owner: realtime -- name: Build release - # IF A BUILD EXISTS the following message is shown - # Release realtime-0.7.1 already exists. Overwrite? [Yn] - # - # There is no flag to stop if an existing build is available, thus we pipe 'n' to stop - shell: echo n | MIX_ENV=prod mix release - become_user: realtime - args: - chdir: /opt/realtime/server +- name: Create /etc/realtime + file: + path: /etc/realtime + state: directory + owner: realtime -- name: Create /etc/realtime.env +- name: Dump /etc/realtime/realtime.env copy: src: files/realtime.env - dest: /etc/realtime.env + dest: /etc/realtime/realtime.env owner: realtime - name: Create service file @@ -46,6 +37,3 @@ daemon_reload: yes name: realtime enabled: yes - -- name: Restart service - service: name=realtime state=restarted diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 7d54b701b..bd22eb731 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -1,7 +1,28 @@ # DigitalOcean's ubuntu droplet isn't up to date with installed packages, and on # a fresh install I see 71 security upgrades available. - name: System - apt update and apt upgrade - apt: update_cache=yes upgrade=yes # SEE http://archive.vn/DKJjs#parameter-upgrade + apt: + update_cache=yes upgrade=yes + # SEE http://archive.vn/DKJjs#parameter-upgrade + +- name: add universe repository for bionic + apt_repository: + repo: deb http://archive.ubuntu.com/ubuntu bionic universe + state: present + +- name: Install essentials + apt: + pkg: + - ufw + - fail2ban + - unattended-upgrades + update_cache: yes + cache_valid_time: 3600 + +- name: Adjust APT update intervals + copy: + src: files/apt_periodic + dest: /etc/apt/apt.conf.d/10periodic - name: System - Create services.slice template: @@ -10,3 +31,26 @@ - name: System - systemd reload systemd: daemon_reload=yes + +- name: UFW - Deny incoming traffics by default + ufw: + state: enabled + default: deny + direction: incoming + +- name: UFW - Allow SSH + ufw: + rule: allow + name: OpenSSH + +- name: UFW - Allow Postgres + ufw: + rule: allow + port: "5432" + proto: tcp + +- name: UFW - Allow realtime + ufw: + rule: allow + port: "4000" + proto: tcp diff --git a/aws.json b/aws.json index 3adec6813..9aa4439ee 100644 --- a/aws.json +++ b/aws.json @@ -1,24 +1,19 @@ { "variables": { "aws_access_key": "{{env `AWS_ACCESS_KEY`}}", - "aws_secret_key": "{{env `AWS_SECRET_KEY`}}" + "aws_secret_key": "{{env `AWS_SECRET_KEY`}}", + "region": "ap-southeast-1", + "source_ami": "ami-0e763a959ec839f5e", + "instance_type": "t2.micro" }, "builders": [ { "type": "amazon-ebs", "access_key": "{{user `aws_access_key`}}", "secret_key": "{{user `aws_secret_key`}}", - "region": "ap-southeast-1", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "name": "ubuntu/images/*ubuntu-bionic-18.04-amd64-server-*", - "root-device-type": "ebs" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "instance_type": "t2.micro", + "region": "{{user `region`}}", + "source_ami": "{{user `source_ami`}}", + "instance_type": "{{user `instance_type`}}", "ssh_username": "ubuntu", "ami_name": "supabase-realtime-0.7.4" } @@ -26,7 +21,20 @@ "provisioners": [ { "type": "ansible", - "playbook_file": "ansible/playbook.yml" + "playbook_file": "ansible/playbook.yml", + "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"], + "user": "ubuntu" + }, + { + "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", + "type": "shell", + "scripts": [ + "scripts/01-test", + "scripts/02-credentials_cleanup.sh", + "scripts/90-cleanup.sh", + "scripts/91-log_cleanup.sh", + "scripts/99-img_check.sh" + ] } ] } diff --git a/do.json b/do.json index 1f9af31bd..a762b0f43 100644 --- a/do.json +++ b/do.json @@ -1,14 +1,16 @@ { "variables": { - "do_api_token": "{{env `DO_API_TOKEN`}}" + "do_api_token": "{{env `DO_API_TOKEN`}}", + "region": "sgp1", + "size": "512mb" }, "builders": [ { "type": "digitalocean", "api_token": "{{user `do_api_token`}}", "image": "ubuntu-18-04-x64", - "region": "sgp1", - "size": "512mb", + "region": "{{user `region`}}", + "size": "{{user `size`}}", "ssh_username": "root", "snapshot_name": "supabase-realtime-0.7.4" } @@ -17,7 +19,17 @@ { "type": "ansible", "playbook_file": "ansible/playbook.yml", - "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"] + "ansible_env_vars": ["ANSIBLE_SSH_ARGS='-o IdentitiesOnly=yes'"], + "user": "root" + }, + { + "type": "shell", + "scripts": [ + "scripts/01-test", + "scripts/90-cleanup.sh", + "scripts/91-log_cleanup.sh", + "scripts/99-img_check.sh" + ] } ] } diff --git a/scripts/01-test b/scripts/01-test new file mode 100644 index 000000000..e5b3e0523 --- /dev/null +++ b/scripts/01-test @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Scripts in this directory are run during the build process. +# each script will be uploaded to /tmp on your build droplet, +# given execute permissions and run. The cleanup process will +# remove the scripts from your build system after they have run +# if you use the build_image task. +# +echo "Commencing Digital Ocean Checks" diff --git a/scripts/02-credentials_cleanup.sh b/scripts/02-credentials_cleanup.sh new file mode 100644 index 000000000..a7b966f03 --- /dev/null +++ b/scripts/02-credentials_cleanup.sh @@ -0,0 +1 @@ +sudo rm /home/ubuntu/.ssh/authorized_keys diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh new file mode 100644 index 000000000..98ab5106d --- /dev/null +++ b/scripts/90-cleanup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +apt-get -y update +apt-get -y upgrade +rm -rf /tmp/* /var/tmp/* +history -c +cat /dev/null > /root/.bash_history +unset HISTFILE +apt-get -y autoremove +apt-get -y autoclean +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +rm -rf /var/lib/cloud/instances/* +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +# Securely erase the unused portion of the filesystem +GREEN='\033[0;32m' +NC='\033[0m' +printf "\n${GREEN}Writing zeros to the remaining disk space to securely +erase the unused portion of the file system. +Depending on your disk size this may take several minutes. +The secure erase will complete successfully when you see:${NC} + dd: writing to '/zerofile': No space left on device\n +Beginning secure erase now\n" + +dd if=/dev/zero of=/zerofile & + PID=$! + while [ -d /proc/$PID ] + do + printf "." + sleep 5 + done +sync; rm /zerofile; sync +cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh new file mode 100644 index 000000000..26f5fbcc9 --- /dev/null +++ b/scripts/91-log_cleanup.sh @@ -0,0 +1,5 @@ +#!/bin/bash +#Erasing all logs +# +echo "Clearing all log files" +rm -rf /var/log/* diff --git a/scripts/99-img_check.sh b/scripts/99-img_check.sh new file mode 100755 index 000000000..6daee6871 --- /dev/null +++ b/scripts/99-img_check.sh @@ -0,0 +1,676 @@ +#!/bin/bash +# +# DigitalOcean Marketplace Image Validation Tool +# © 2018 DigitalOcean LLC. +# This code is licensed under MIT license (see LICENSE.txt for details) +# +VERSION="v. 1.2" +RUNDATE=$( date ) + +# Script should be run with SUDO +if [ "$EUID" -ne 0 ] + then echo "[Error] - This script must be run with sudo or as the root user." + exit 1 +fi + +STATUS=0 +PASS=0 +WARN=0 +FAIL=0 + +# $1 == command to check for +# returns: 0 == true, 1 == false +cmdExists() { + if command -v "$1" > /dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +function getDistro { + if [ -f /etc/os-release ]; then + # freedesktop.org and systemd + . /etc/os-release + OS=$NAME + VER=$VERSION_ID +elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + OS=$(lsb_release -si) + VER=$(lsb_release -sr) +elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + OS=$DISTRIB_ID + VER=$DISTRIB_RELEASE +elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + VER=$(cat /etc/debian_version) +elif [ -f /etc/SuSe-release ]; then + # Older SuSE/etc. + : +elif [ -f /etc/redhat-release ]; then + # Older Red Hat, CentOS, etc. + VER=$( cat /etc/redhat-release | cut -d" " -f3 | cut -d "." -f1) + d=$( cat /etc/redhat-release | cut -d" " -f1 | cut -d "." -f1) + if [[ $d == "CentOS" ]]; then + OS="CentOS Linux" + fi +else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + VER=$(uname -r) +fi +} +function loadPasswords { +SHADOW=$(cat /etc/shadow) +} + +function checkAgent { + # Check for the presence of the do-agent in the filesystem + if [ -d /var/opt/digitalocean/do-agent ];then + echo -en "\e[41m[FAIL]\e[0m DigitalOcean Monitoring Agent detected.\n" + ((FAIL++)) + STATUS=2 + if [[ $OS == "CentOS Linux" ]]; then + echo "The agent can be removed with 'sudo yum remove do-agent' " + elif [[ $OS == "Ubuntu" ]]; then + echo "The agent can be removed with 'sudo apt-get purge do-agent' " + fi + else + echo -en "\e[32m[PASS]\e[0m DigitalOcean Monitoring agent was not found\n" + ((PASS++)) + fi +} + +function checkLogs { + cp_ignore="/var/log/cpanel-install.log" + echo -en "\nChecking for log files in /var/log\n\n" + # Check if there are log archives or log files that have not been recently cleared. + for f in /var/log/*-????????; do + [[ -e $f ]] || break + if [ $f != $cp_ignore ]; then + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + done + for f in /var/log/*.[0-9];do + [[ -e $f ]] || break + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + done + for f in /var/log/*.log; do + [[ -e $f ]] || break + if [[ "${f}" = '/var/log/lfd.log' && "$( cat "${f}" | egrep -v '/var/log/messages has been reset| Watching /var/log/messages' | wc -c)" -gt 50 ]]; then + if [ $f != $cp_ignore ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [[ "${f}" == '/var/log/cloud-init-output.log' ]]; then + if cat '/var/log/cloud-init-output.log' | grep -q SHA256; then + echo -en "\e[41m[FAIL]\e[0m log containing SHA256 value found in log file ${f}\n" + ((FAIL++)) + STATUS=1 + fi + elif [[ "${f}" != '/var/log/lfd.log' && "$( cat "${f}" | wc -c)" -gt 50 ]]; then + if [ $f != $cp_ignore ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + done +} +function checkTMP { + # Check the /tmp directory to ensure it is empty. Warn on any files found. + return 1 +} +function checkRoot { + user="root" + uhome="/root" + for usr in $SHADOW + do + IFS=':' read -r -a u <<< "$usr" + if [[ "${u[0]}" == "${user}" ]]; then + if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then + echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account.\n" + ((FAIL++)) + STATUS=2 + fi + fi + done + if [ -d ${uhome}/ ]; then + if [ -d ${uhome}/.ssh/ ]; then + if ls ${uhome}/.ssh/*> /dev/null 2>&1; then + for key in ${uhome}/.ssh/* + do + if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then + + if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" + akey=$(cat ${key}) + echo "File Contents:" + echo $akey + echo "--------------" + ((FAIL++)) + STATUS=2 + fi + elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then + if [ "$( cat "${key}" | wc -c)" -gt 0 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" + akey=$(cat ${key}) + echo "File Contents:" + echo $akey + echo "--------------" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory at \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + else + if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a populated known_hosts file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + done + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" + fi + if [ -f /root/.bash_history ];then + + BH_S=$( cat /root/.bash_history | wc -c) + + if [[ $BH_S -lt 200 ]]; then + echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m ${user}'s Bash History should be cleared to prevent sensitive information from leaking\n" + ((FAIL++)) + STATUS=2 + fi + + return 1; + else + echo -en "\e[32m[PASS]\e[0m The Root User's Bash History is not present\n" + ((PASS++)) + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have a directory in /home\n" + fi + echo -en "\n\n" + return 1 +} + +function checkUsers { + # Check each user-created account + for user in $(awk -F: '$3 >= 1000 && $1 != "nobody" {print $1}' /etc/passwd;) + do + # Skip some other non-user system accounts + if [[ $user == "centos" ]]; then + : + elif [[ $user == "nfsnobody" ]]; then + : + else + echo -en "\nChecking user: ${user}...\n" + for usr in $SHADOW + do + IFS=':' read -r -a u <<< "$usr" + if [[ "${u[0]}" == "${user}" ]]; then + if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then + echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account.\n" + ((FAIL++)) + STATUS=2 + fi + fi + done + #echo "User Found: ${user}" + uhome="/home/${user}" + if [ -d "${uhome}/" ]; then + if [ -d "${uhome}/.ssh/" ]; then + if ls "${uhome}/.ssh/*"> /dev/null 2>&1; then + for key in ${uhome}/.ssh/* + do + if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then + if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" + akey=$(cat ${key}) + echo "File Contents:" + echo $akey + echo "--------------" + ((FAIL++)) + STATUS=2 + fi + elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then + if [ "$( cat "${key}" | wc -c)" -gt 0 ]; then + echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" + akey=$(cat ${key}) + echo "File Contents:" + echo $akey + echo "--------------" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + elif [ "${key}" != "${uhome}/.ssh/known_hosts" ]; then + + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a file in their .ssh directory named \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + + else + if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a known_hosts file in \e[93m${key}\e[0m\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 + fi + fi + fi + + + done + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m has no SSH keys present\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have an .ssh directory\n" + fi + else + echo -en "\e[32m[ OK ]\e[0m User \e[1m${user}\e[0m does not have a directory in /home\n" + fi + + # Check for an uncleared .bash_history for this user + if [ -f "${uhome}/.bash_history" ]; then + BH_S=$( cat "${uhome}/.bash_history" | wc -c ) + + if [[ $BH_S -lt 200 ]]; then + echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" + ((PASS++)) + else + echo -en "\e[41m[FAIL]\e[0m ${user}'s Bash History should be cleared to prevent sensitive information from leaking\n" + ((FAIL++)) + STATUS=2 + + fi + echo -en "\n\n" + fi + fi + done +} +function checkFirewall { + + if [[ $OS == "Ubuntu" ]]; then + fw="ufw" + ufwa=$(ufw status |head -1| sed -e "s/^Status:\ //") + if [[ $ufwa == "active" ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + elif [[ $OS == "CentOS Linux" ]]; then + if [ -f /usr/lib/systemd/system/csf.service ]; then + fw="csf" + if [[ $(systemctl status $fw >/dev/null 2>&1) ]]; then + + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + elif cmdExists "firewall-cmd"; then + if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + fw="firewalld" + if [[ $(systemctl is-active firewalld >/dev/null 2>&1 && echo 1 || echo 0) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + fi + elif [[ "$OS" =~ Debian.* ]]; then + # user could be using a number of different services for managing their firewall + # we will check some of the most common + if cmdExists 'ufw'; then + fw="ufw" + ufwa=$(ufw status | sed -e "s/^Status:\ //") + if [[ $ufwa == "active" ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + elif cmdExists "firewall-cmd"; then + fw="firewalld" + if [[ $(systemctl is-active --quiet $fw) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + else + # user could be using vanilla iptables, check if kernel module is loaded + fw="iptables" + if [[ $(lsmod | grep -q '^ip_tables' 2>/dev/null) ]]; then + FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + ((PASS++)) + else + FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + ((WARN++)) + fi + fi + fi + +} +function checkUpdates { + if [[ $OS == "Ubuntu" ]] || [[ "$OS" =~ Debian.* ]]; then + echo -en "\nUpdating apt package database to check for security updates, this may take a minute...\n\n" + apt-get -y update > /dev/null + + uc=$(apt-get --just-print upgrade | grep -i "security" | wc -l) + if [[ $uc -gt 0 ]]; then + update_count=$(( ${uc} / 2 )) + else + update_count=0 + fi + + if [[ $update_count -gt 0 ]]; then + echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" + echo -en + echo -en "Here is a list of the security updates that are not installed:\n" + sleep 2 + apt-get --just-print upgrade | grep -i security | awk '{print $2}' | awk '!seen[$0]++' + echo -en + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n\n" + fi + elif [[ $OS == "CentOS Linux" ]]; then + echo -en "\nChecking for available updates with yum, this may take a minute...\n\n" + + update_count=$(yum list updates -q | grep -vc "Updated Packages") + if [[ $update_count -gt 0 ]]; then + echo -en "\e[41m[FAIL]\e[0m There are ${update_count} updates available for this image that have not been installed.\n" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n" + ((PASS++)) + fi + else + echo "Error encountered" + exit 1 + fi + + return 1; +} +function checkCloudInit { + + if hash cloud-init 2>/dev/null; then + CI="\e[32m[PASS]\e[0m Cloud-init is installed.\n" + ((PASS++)) + else + CI="\e[41m[FAIL]\e[0m No valid verison of cloud-init was found.\n" + ((FAIL++)) + STATUS=2 + fi + return 1 +} +function checkMongoDB { + # Check if MongoDB is installed + # If it is, verify the version is allowed (non-SSPL) + + if [[ $OS == "Ubuntu" ]] || [[ "$OS" =~ Debian.* ]]; then + + if [[ -f "/usr/bin/mongod" ]]; then + version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") + + if version_gt $version 4.0.0; then + if version_gt $version 4.0.3; then + echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" + ((PASS++)) + fi + else + if version_gt $version 3.6.8; then + echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" + ((PASS++)) + fi + fi + + + else + echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" + ((PASS++)) + fi + + elif [[ $OS == "CentOS Linux" ]]; then + + if [[ -f "/usr/bin/mongod" ]]; then + version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") + + + if version_gt $version 4.0.0; then + if version_gt $version 4.0.3; then + echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" + ((PASS++)) + fi + else + if version_gt $version 3.6.8; then + echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" + ((FAIL++)) + STATUS=2 + else + echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" + ((PASS++)) + fi + fi + + + + else + echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" + ((PASS++)) + fi + + else + echo "ERROR: Unable to identify distribution" + ((FAIL++)) + STATUS 2 + return 1 + fi + + +} + +function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + + +clear +echo "DigitalOcean Marketplace Image Validation Tool ${VERSION}" +echo "Executed on: ${RUNDATE}" +echo "Checking local system for Marketplace compatibility..." + +getDistro + +echo -en "\n\e[1mDistribution:\e[0m ${OS}\n" +echo -en "\e[1mVersion:\e[0m ${VER}\n\n" + +ost=0 +osv=0 + +if [[ $OS == "Ubuntu" ]]; then + ost=1 + if [[ $VER == "18.04" ]]; then + osv=1 + elif [[ $VER == "16.04" ]]; then + osv=1 + else + osv=0 + fi + +elif [[ "$OS" =~ Debian.* ]]; then + ost=1 + case "$VER" in + 9) + osv=1 + ;; + 10) + osv=1 + ;; + *) + osv=2 + ;; + esac + +elif [[ $OS == "CentOS Linux" ]]; then + ost=1 + if [[ $VER == "7" ]]; then + osv=1 + elif [[ $VER == "6" ]]; then + osv=1 + else + osv=2 + fi +else + ost=0 +fi + +if [[ $ost == 1 ]]; then + echo -en "\e[32m[PASS]\e[0m Supported Operating System Detected: ${OS}\n" + ((PASS++)) +else + echo -en "\e[41m[FAIL]\e[0m ${OS} is not a supported Operating System\n" + ((FAIL++)) + STATUS=2 +fi + +if [[ $osv == 1 ]]; then + echo -en "\e[32m[PASS]\e[0m Supported Release Detected: ${VER}\n" + ((PASS++)) +elif [[ $ost == 1 ]]; then + echo -en "\e[41m[FAIL]\e[0m ${OS} ${VER} is not a supported Operating System Version\n" + ((FAIL++)) + STATUS=2 +else + echo "Exiting..." + exit 1 +fi + +checkCloudInit + +echo -en "${CI}" + +checkFirewall + +echo -en "${FW_VER}" + +checkUpdates + +loadPasswords + +checkLogs + +echo -en "\n\nChecking all user-created accounts...\n" +checkUsers + +echo -en "\n\nChecking the root account...\n" +checkRoot + +checkAgent + +checkMongoDB + + +# Summary +echo -en "\n\n---------------------------------------------------------------------------------------------------\n" + +if [[ $STATUS == 0 ]]; then + echo -en "Scan Complete.\n\e[32mAll Tests Passed!\e[0m\n" +elif [[ $STATUS == 1 ]]; then + echo -en "Scan Complete. \n\e[93mSome non-critical tests failed. Please review these items.\e[0m\e[0m\n" +else + echo -en "Scan Complete. \n\e[41mOne or more tests failed. Please review these items and re-test.\e[0m\n" +fi +echo "---------------------------------------------------------------------------------------------------" +echo -en "\e[1m${PASS} Tests PASSED\e[0m\n" +echo -en "\e[1m${WARN} WARNINGS\e[0m\n" +echo -en "\e[1m${FAIL} Tests FAILED\e[0m\n" +echo -en "---------------------------------------------------------------------------------------------------\n" + +if [[ $STATUS == 0 ]]; then + echo -en "We did not detect any issues with this image. Please be sure to manually ensure that all software installed on the base system is functional, secure and properly configured (or facilities for configuration on first-boot have been created).\n\n" + exit 0 +elif [[ $STATUS == 1 ]]; then + echo -en "Please review all [WARN] items above and ensure they are intended or resolved. If you do not have a specific requirement, we recommend resolving these items before image submission\n\n" + exit 1 +else + echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the marketplace.\n\n" + exit 1 +fi diff --git a/server/config/config.exs b/server/config/config.exs index 6f311ec2f..66509150e 100644 --- a/server/config/config.exs +++ b/server/config/config.exs @@ -10,7 +10,6 @@ use Mix.Config # Configures the endpoint config :realtime, RealtimeWeb.Endpoint, url: [host: "localhost"], - secret_key_base: "+Cu2K/kBuuKLyAidsgfktbQkBuuvXPHA1NxWWZz8yXM0n+2fyZ305QQ0AibOEagw", render_errors: [view: RealtimeWeb.ErrorView, accepts: ~w(html json)], pubsub: [name: Realtime.PubSub, adapter: Phoenix.PubSub.PG2] diff --git a/server/config/releases.exs b/server/config/releases.exs index 3f0d570fe..d74a40476 100644 --- a/server/config/releases.exs +++ b/server/config/releases.exs @@ -9,8 +9,17 @@ db_port = System.get_env("DB_PORT") || 5432 db_name = System.get_env("DB_NAME") || "postgres" db_ssl = System.get_env("DB_SSL") || true +secret_key_base = + System.get_env("SECRET_KEY_BASE") || + raise """ + environment variable SECRET_KEY_BASE is missing. + You can generate one by calling: + export SECRET_KEY_BASE=`openssl rand -base64 48` + """ + config :realtime, RealtimeWeb.Endpoint, - http: [:inet6, port: String.to_integer(app_port)] + http: [:inet6, port: String.to_integer(app_port)], + secret_key_base: secret_key_base config :realtime, app_port: app_port