Introduce encryption - GnuPG

Dockerfile

@@ -19,7 +19,8 @@ ARG VCS_REF
 ARG VERSION
 
 ENV MONGODB_TOOLS_VERSION 4.2.1-r1
-ENV GOOGLE_CLOUD_SDK_VERSION 276.0.0
+ENV GNUPG_VERSION 2.2.19-r0
+ENV GOOGLE_CLOUD_SDK_VERSION 315.0.0
 ENV AZURE_CLI_VERSION 2.13.0
 ENV AWS_CLI_VERSION 1.18.159
 ENV PATH /root/google-cloud-sdk/bin:$PATH
@@ -34,7 +35,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.version=$VERSION \
       org.label-schema.schema-version="1.0"
 
-RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERSION}
+RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERSION} gnupg=${GNUPG_VERSION}
 ADD https://dl.minio.io/client/mc/release/linux-amd64/mc /usr/bin
 RUN chmod u+x /usr/bin/mc
 
@@ -71,7 +72,7 @@ RUN apk --no-cache add \
 # install azure-cli and aws-cli
 RUN apk --no-cache add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \
   pip --no-cache-dir install cffi && \
-  pip --no-cache-dir install azure-cli==${AZURE_CLI_VERSION} && \
+  pip --no-cache-dir --use-feature=2020-resolver install azure-cli==${AZURE_CLI_VERSION} && \
   pip --no-cache-dir install awscli==${AWS_CLI_VERSION} && \
   apk del --purge build
 

README.md

@@ -79,6 +79,18 @@ target:
   password: "secret"
   # add custom params to mongodump (eg. Auth or SSL support), leave blank if not needed
   params: "--ssl --authenticationDatabase admin"
+# Encryption (optional)
+encryption:
+  # At the time being, only gpg asymmetric encryption is supported
+  # Public key file or at least one recipient is mandatory
+  gpg:
+    # optional path to a public key file, only the first key is used.
+    keyFile: /secret/mgob-key/key.pub
+    # optional key server, defaults to hkps://keys.openpgp.org
+    keyServer: hkps://keys.openpgp.org
+    # optional list of recipients, they will be looked up on key server
+    recipients:
+    - example@example.com
 # S3 upload (optional)
 s3:
   url: "https://play.minio.io:9000"

cmd/mgob/mgob.go

@@ -18,7 +18,7 @@ import (
 
 var (
 	appConfig = &config.AppConfig{}
-	version   = "v1.2.0-dev"
+	version   = "v1.3.0-dev"
 )
 
 func beforeApp(c *cli.Context) error {
@@ -95,10 +95,12 @@ func start(c *cli.Context) error {
 	appConfig.TmpPath = c.String("TmpPath")
 	appConfig.DataPath = c.String("DataPath")
 	appConfig.Version = version
-	appConfig.UseAwsCli = true
 
 	log.Infof("starting with config: %+v", appConfig)
 
+	appConfig.UseAwsCli = true
+	appConfig.HasGpg = true
+
 	info, err := backup.CheckMongodump()
 	if err != nil {
 		log.Fatal(err)
@@ -118,6 +120,13 @@ func start(c *cli.Context) error {
 	}
 	log.Info(info)
 
+	info, err = backup.CheckGpg()
+	if err != nil {
+		log.Warn(err)
+		appConfig.HasGpg = false
+	}
+	log.Info(info)
+
 	info, err = backup.CheckGCloudClient()
 	if err != nil {
 		log.Fatal(err)

pkg/backup/backup.go

@@ -73,6 +73,18 @@ func Run(plan config.Plan, conf *config.AppConfig) (Result, error) {
 
 	file := filepath.Join(planDir, res.Name)
 
+	if plan.Encryption != nil {
+		encryptedFile := fmt.Sprintf("%v.encrypted", file)
+		output, err := encrypt(file, encryptedFile, plan, conf)
+		if err != nil {
+			return res, err
+		} else {
+			removeUnencrypted(file, encryptedFile)
+			file = encryptedFile
+			log.WithField("plan", plan.Name).Infof("Encryption finished %v", output)
+		}
+	}
+
 	if plan.SFTP != nil {
 		sftpOutput, err := sftpUpload(file, plan)
 		if err != nil {

pkg/backup/checks.go

@@ -46,6 +46,19 @@ func CheckAWSClient() (string, error) {
 	return strings.Replace(string(output), "\n", " ", -1), nil
 }
 
+func CheckGpg() (string, error) {
+	output, err := sh.Command("/bin/sh", "-c", "gpg --version").CombinedOutput()
+	if err != nil {
+		ex := ""
+		if len(output) > 0 {
+			ex = strings.Replace(string(output), "\n", " ", -1)
+		}
+		return "", errors.Wrapf(err, "gpg failed %v", ex)
+	}
+
+	return strings.Replace(string(output), "\n", " ", -1), nil
+}
+
 func CheckGCloudClient() (string, error) {
 	output, err := sh.Command("/bin/sh", "-c", "gcloud --version").CombinedOutput()
 	if err != nil {

pkg/backup/encrypt.go

@@ -0,0 +1,92 @@
+package backup
+
+import (
+	"fmt"
+	"github.com/codeskyblue/go-sh"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/stefanprodan/mgob/pkg/config"
+	"os"
+	"regexp"
+	"strings"
+)
+
+func encrypt(file string, encryptedFile string, plan config.Plan, conf *config.AppConfig) (string, error) {
+	if plan.Encryption.Gpg != nil {
+		if !conf.HasGpg {
+			return "", errors.Errorf("GPG configuration is present, but no GPG binary is found! Uploading unencrypted backup.")
+		}
+		return gpgEncrypt(file, encryptedFile, plan)
+	}
+
+	return "", errors.Errorf("Encryption config is not valid!")
+}
+
+func removeUnencrypted(file string, encryptedFile string) {
+	// Check if encrypted file exists and remove original
+	stat, err := os.Stat(encryptedFile)
+	if err == nil && stat.Size() > 0 {
+		os.Remove(file)
+	}
+}
+
+func gpgEncrypt(file string, encryptedFile string, plan config.Plan) (string, error) {
+	output := ""
+	recipient := ""
+
+	recipients := plan.Encryption.Gpg.Recipients
+
+	keyFile := plan.Encryption.Gpg.KeyFile
+	if keyFile != "" {
+		keyFileStat, err := os.Stat(keyFile)
+		if err == nil && !keyFileStat.IsDir() {
+			// import key from file
+			importCmd := fmt.Sprintf("gpg --batch --import  %v", keyFile)
+
+			result, err := sh.Command("/bin/sh", "-c", importCmd).CombinedOutput()
+			if len(result) > 0 {
+				output += strings.Replace(string(result), "\n", " ", -1)
+			}
+			if err != nil {
+				return "", errors.Wrapf(err, "Importing encryption key for plan %v failed %s", plan.Name, output)
+			}
+			if !strings.Contains(output, "imported: 1") && !strings.Contains(output, "unchanged: 1") {
+				return "", errors.Errorf("Importing encryption key failed %v", output)
+			}
+
+			re := regexp.MustCompile(`key ([0-9A-F]+):`)
+			keyMatch := re.FindStringSubmatch(output)
+			log.WithField("plan", plan.Name).Debugf("Import output: %v", output)
+			log.WithField("plan", plan.Name).Debugf("Parsed key id: %v", keyMatch[1])
+			if keyMatch != nil {
+				recipients = append(recipients, keyMatch[1])
+			}
+		}
+	}
+
+	recipient = strings.Join(recipients, " -r ")
+
+	if recipient == "" {
+		return "", errors.Errorf("GPG configuration is present, but no encryption key is configured! %v", output)
+	}
+
+	keyServer := plan.Encryption.Gpg.KeyServer
+	if keyServer == "" {
+		keyServer = "hkps://keys.openpgp.org"
+	}
+
+	// encrypt file
+	encryptCmd := fmt.Sprintf(
+		"gpg -v --batch --yes --trust-model always --auto-key-locate local,%v -e -r %v -o %v %v",
+		keyServer, recipient, encryptedFile, file)
+
+	result, err := sh.Command("/bin/sh", "-c", encryptCmd).CombinedOutput()
+	if len(result) > 0 {
+		output += strings.Replace(string(result), "\n", " ", -1)
+	}
+	if err != nil {
+		return "", errors.Wrapf(err, "Encryption for plan %v failed %s", plan.Name, output)
+	}
+
+	return output, nil
+}

pkg/backup/local.go

@@ -67,7 +67,7 @@ func logToFile(file string, data []byte) error {
 }
 
 func applyRetention(path string, retention int) error {
-	gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz | tail -n +%v)", path, retention+1)
+	gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz *.gz.encrypted | tail -n +%v)", path, retention+1)
 	err := sh.Command("/bin/sh", "-c", gz).Run()
 	if err != nil {
 		return errors.Wrapf(err, "removing old gz files from %v failed", path)

pkg/config/app.go

@@ -10,4 +10,5 @@ type AppConfig struct {
 	DataPath    string `json:"data_path"`
 	Version     string `json:"version"`
 	UseAwsCli   bool   `json:"use_aws_cli"`
+	HasGpg      bool   `json:"has_gpg"`
 }

pkg/config/plan.go

@@ -11,16 +11,17 @@ import (
 )
 
 type Plan struct {
-	Name      string    `yaml:"name"`
-	Target    Target    `yaml:"target"`
-	Scheduler Scheduler `yaml:"scheduler"`
-	S3        *S3       `yaml:"s3"`
-	GCloud    *GCloud   `yaml:"gcloud"`
-	Rclone    *Rclone   `yaml:"rclone"`
-	Azure     *Azure    `yaml:"azure"`
-	SFTP      *SFTP     `yaml:"sftp"`
-	SMTP      *SMTP     `yaml:"smtp"`
-	Slack     *Slack    `yaml:"slack"`
+	Name       string      `yaml:"name"`
+	Target     Target      `yaml:"target"`
+	Scheduler  Scheduler   `yaml:"scheduler"`
+	Encryption *Encryption `yaml:"encryption"`
+	S3         *S3         `yaml:"s3"`
+	GCloud     *GCloud     `yaml:"gcloud"`
+	Rclone     *Rclone     `yaml:"rclone"`
+	Azure      *Azure      `yaml:"azure"`
+	SFTP       *SFTP       `yaml:"sftp"`
+	SMTP       *SMTP       `yaml:"smtp"`
+	Slack      *Slack      `yaml:"slack"`
 }
 
 type Target struct {
@@ -39,6 +40,16 @@ type Scheduler struct {
 	Timeout   int    `yaml:"timeout"`
 }
 
+type Encryption struct {
+	Gpg *Gpg `yaml:"gpg"`
+}
+
+type Gpg struct {
+	KeyServer  string   `yaml:"keyServer"`
+	Recipients []string `yaml:"recipients"`
+	KeyFile    string   `yaml:"keyFile"`
+}
+
 type S3 struct {
 	Bucket       string `yaml:"bucket"`
 	AccessKey    string `yaml:"accessKey"`