Skip to content
This repository has been archived by the owner on Nov 3, 2022. It is now read-only.

Commit

Permalink
Merge pull request #122 from endrec/gpg
Browse files Browse the repository at this point in the history
Introduce encryption - GnuPG
  • Loading branch information
stefanprodan authored Nov 3, 2020
2 parents 1a40e5a + 740e2a4 commit d9865ee
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 16 deletions.
7 changes: 4 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ ARG VCS_REF
ARG VERSION

ENV MONGODB_TOOLS_VERSION 4.2.1-r1
ENV GOOGLE_CLOUD_SDK_VERSION 276.0.0
ENV GNUPG_VERSION 2.2.19-r0
ENV GOOGLE_CLOUD_SDK_VERSION 315.0.0
ENV AZURE_CLI_VERSION 2.13.0
ENV AWS_CLI_VERSION 1.18.159
ENV PATH /root/google-cloud-sdk/bin:$PATH
Expand All @@ -34,7 +35,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.version=$VERSION \
org.label-schema.schema-version="1.0"

RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERSION}
RUN apk add --no-cache ca-certificates tzdata mongodb-tools=${MONGODB_TOOLS_VERSION} gnupg=${GNUPG_VERSION}
ADD https://dl.minio.io/client/mc/release/linux-amd64/mc /usr/bin
RUN chmod u+x /usr/bin/mc

Expand Down Expand Up @@ -71,7 +72,7 @@ RUN apk --no-cache add \
# install azure-cli and aws-cli
RUN apk --no-cache add --virtual=build gcc libffi-dev musl-dev openssl-dev python3-dev make && \
pip --no-cache-dir install cffi && \
pip --no-cache-dir install azure-cli==${AZURE_CLI_VERSION} && \
pip --no-cache-dir --use-feature=2020-resolver install azure-cli==${AZURE_CLI_VERSION} && \
pip --no-cache-dir install awscli==${AWS_CLI_VERSION} && \
apk del --purge build

Expand Down
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,18 @@ target:
password: "secret"
# add custom params to mongodump (eg. Auth or SSL support), leave blank if not needed
params: "--ssl --authenticationDatabase admin"
# Encryption (optional)
encryption:
# At the time being, only gpg asymmetric encryption is supported
# Public key file or at least one recipient is mandatory
gpg:
# optional path to a public key file, only the first key is used.
keyFile: /secret/mgob-key/key.pub
# optional key server, defaults to hkps://keys.openpgp.org
keyServer: hkps://keys.openpgp.org
# optional list of recipients, they will be looked up on key server
recipients:
- example@example.com
# S3 upload (optional)
s3:
url: "https://play.minio.io:9000"
Expand Down
13 changes: 11 additions & 2 deletions cmd/mgob/mgob.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (

var (
appConfig = &config.AppConfig{}
version = "v1.2.0-dev"
version = "v1.3.0-dev"
)

func beforeApp(c *cli.Context) error {
Expand Down Expand Up @@ -95,10 +95,12 @@ func start(c *cli.Context) error {
appConfig.TmpPath = c.String("TmpPath")
appConfig.DataPath = c.String("DataPath")
appConfig.Version = version
appConfig.UseAwsCli = true

log.Infof("starting with config: %+v", appConfig)

appConfig.UseAwsCli = true
appConfig.HasGpg = true

info, err := backup.CheckMongodump()
if err != nil {
log.Fatal(err)
Expand All @@ -118,6 +120,13 @@ func start(c *cli.Context) error {
}
log.Info(info)

info, err = backup.CheckGpg()
if err != nil {
log.Warn(err)
appConfig.HasGpg = false
}
log.Info(info)

info, err = backup.CheckGCloudClient()
if err != nil {
log.Fatal(err)
Expand Down
12 changes: 12 additions & 0 deletions pkg/backup/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@ func Run(plan config.Plan, conf *config.AppConfig) (Result, error) {

file := filepath.Join(planDir, res.Name)

if plan.Encryption != nil {
encryptedFile := fmt.Sprintf("%v.encrypted", file)
output, err := encrypt(file, encryptedFile, plan, conf)
if err != nil {
return res, err
} else {
removeUnencrypted(file, encryptedFile)
file = encryptedFile
log.WithField("plan", plan.Name).Infof("Encryption finished %v", output)
}
}

if plan.SFTP != nil {
sftpOutput, err := sftpUpload(file, plan)
if err != nil {
Expand Down
13 changes: 13 additions & 0 deletions pkg/backup/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,19 @@ func CheckAWSClient() (string, error) {
return strings.Replace(string(output), "\n", " ", -1), nil
}

func CheckGpg() (string, error) {
output, err := sh.Command("/bin/sh", "-c", "gpg --version").CombinedOutput()
if err != nil {
ex := ""
if len(output) > 0 {
ex = strings.Replace(string(output), "\n", " ", -1)
}
return "", errors.Wrapf(err, "gpg failed %v", ex)
}

return strings.Replace(string(output), "\n", " ", -1), nil
}

func CheckGCloudClient() (string, error) {
output, err := sh.Command("/bin/sh", "-c", "gcloud --version").CombinedOutput()
if err != nil {
Expand Down
92 changes: 92 additions & 0 deletions pkg/backup/encrypt.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package backup

import (
"fmt"
"github.com/codeskyblue/go-sh"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/stefanprodan/mgob/pkg/config"
"os"
"regexp"
"strings"
)

func encrypt(file string, encryptedFile string, plan config.Plan, conf *config.AppConfig) (string, error) {
if plan.Encryption.Gpg != nil {
if !conf.HasGpg {
return "", errors.Errorf("GPG configuration is present, but no GPG binary is found! Uploading unencrypted backup.")
}
return gpgEncrypt(file, encryptedFile, plan)
}

return "", errors.Errorf("Encryption config is not valid!")
}

func removeUnencrypted(file string, encryptedFile string) {
// Check if encrypted file exists and remove original
stat, err := os.Stat(encryptedFile)
if err == nil && stat.Size() > 0 {
os.Remove(file)
}
}

func gpgEncrypt(file string, encryptedFile string, plan config.Plan) (string, error) {
output := ""
recipient := ""

recipients := plan.Encryption.Gpg.Recipients

keyFile := plan.Encryption.Gpg.KeyFile
if keyFile != "" {
keyFileStat, err := os.Stat(keyFile)
if err == nil && !keyFileStat.IsDir() {
// import key from file
importCmd := fmt.Sprintf("gpg --batch --import %v", keyFile)

result, err := sh.Command("/bin/sh", "-c", importCmd).CombinedOutput()
if len(result) > 0 {
output += strings.Replace(string(result), "\n", " ", -1)
}
if err != nil {
return "", errors.Wrapf(err, "Importing encryption key for plan %v failed %s", plan.Name, output)
}
if !strings.Contains(output, "imported: 1") && !strings.Contains(output, "unchanged: 1") {
return "", errors.Errorf("Importing encryption key failed %v", output)
}

re := regexp.MustCompile(`key ([0-9A-F]+):`)
keyMatch := re.FindStringSubmatch(output)
log.WithField("plan", plan.Name).Debugf("Import output: %v", output)
log.WithField("plan", plan.Name).Debugf("Parsed key id: %v", keyMatch[1])
if keyMatch != nil {
recipients = append(recipients, keyMatch[1])
}
}
}

recipient = strings.Join(recipients, " -r ")

if recipient == "" {
return "", errors.Errorf("GPG configuration is present, but no encryption key is configured! %v", output)
}

keyServer := plan.Encryption.Gpg.KeyServer
if keyServer == "" {
keyServer = "hkps://keys.openpgp.org"
}

// encrypt file
encryptCmd := fmt.Sprintf(
"gpg -v --batch --yes --trust-model always --auto-key-locate local,%v -e -r %v -o %v %v",
keyServer, recipient, encryptedFile, file)

result, err := sh.Command("/bin/sh", "-c", encryptCmd).CombinedOutput()
if len(result) > 0 {
output += strings.Replace(string(result), "\n", " ", -1)
}
if err != nil {
return "", errors.Wrapf(err, "Encryption for plan %v failed %s", plan.Name, output)
}

return output, nil
}
2 changes: 1 addition & 1 deletion pkg/backup/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func logToFile(file string, data []byte) error {
}

func applyRetention(path string, retention int) error {
gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz | tail -n +%v)", path, retention+1)
gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz *.gz.encrypted | tail -n +%v)", path, retention+1)
err := sh.Command("/bin/sh", "-c", gz).Run()
if err != nil {
return errors.Wrapf(err, "removing old gz files from %v failed", path)
Expand Down
1 change: 1 addition & 0 deletions pkg/config/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,5 @@ type AppConfig struct {
DataPath string `json:"data_path"`
Version string `json:"version"`
UseAwsCli bool `json:"use_aws_cli"`
HasGpg bool `json:"has_gpg"`
}
31 changes: 21 additions & 10 deletions pkg/config/plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,17 @@ import (
)

type Plan struct {
Name string `yaml:"name"`
Target Target `yaml:"target"`
Scheduler Scheduler `yaml:"scheduler"`
S3 *S3 `yaml:"s3"`
GCloud *GCloud `yaml:"gcloud"`
Rclone *Rclone `yaml:"rclone"`
Azure *Azure `yaml:"azure"`
SFTP *SFTP `yaml:"sftp"`
SMTP *SMTP `yaml:"smtp"`
Slack *Slack `yaml:"slack"`
Name string `yaml:"name"`
Target Target `yaml:"target"`
Scheduler Scheduler `yaml:"scheduler"`
Encryption *Encryption `yaml:"encryption"`
S3 *S3 `yaml:"s3"`
GCloud *GCloud `yaml:"gcloud"`
Rclone *Rclone `yaml:"rclone"`
Azure *Azure `yaml:"azure"`
SFTP *SFTP `yaml:"sftp"`
SMTP *SMTP `yaml:"smtp"`
Slack *Slack `yaml:"slack"`
}

type Target struct {
Expand All @@ -39,6 +40,16 @@ type Scheduler struct {
Timeout int `yaml:"timeout"`
}

type Encryption struct {
Gpg *Gpg `yaml:"gpg"`
}

type Gpg struct {
KeyServer string `yaml:"keyServer"`
Recipients []string `yaml:"recipients"`
KeyFile string `yaml:"keyFile"`
}

type S3 struct {
Bucket string `yaml:"bucket"`
AccessKey string `yaml:"accessKey"`
Expand Down

0 comments on commit d9865ee

Please sign in to comment.