Start a database from a volume and archive snapshot (#337)

To enable restore and clone use cases, we need a way to start a database from a volume snapshot. An archive snapshot is specified as a [TypedObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#typedobjectreference-v1-core) and is passed as the helm value `database.persistence.dataSourceRef`. If the snapshot is of an archive without a journal, the journal snapshot should be passed in the helm values `database.sm.hotCopy.journalPath.persistence.dataSourceRef` and `database.sm.noHotCopy.journalPath.persistence.dataSourceRef`. If the name of the domain or database has changed, there is code to relocate the file and register the moved archive with the admin service. Unfortunately, this logic cannot differentiate between a snapshot recovered without changing the database and domain names and an SM that was restarted. In order to trigger the registration logic in this case, when taking the snapshot, there needs to be a pre-snapshot script that will create a `backup.txt` file in the archive (and journal, if it is on a separate volume). That file should just contain the backup id as plain text. That same backup id will need to be set as the helm value `database.autoImport.backup_id`. At restore time, the file will be deleted to prevent the archive from being registered twice.
nuodb · Dec 20, 2023 · 913fe72 · 913fe72
1 parent 2e4e6c5
commit 913fe72
Show file tree

Hide file tree

Showing 13 changed files with 897 additions and 668 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -79,7 +79,7 @@ jobs:
         type: string
 
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-2004:2023.07.1
 
     resource_class: <<parameters.resource-class>>
 
@@ -179,7 +179,7 @@ workflows:
           test-suite: "./test/minikube"
           tag: "long"
           requires-minikube: true
-          timeout: "90m"
+          timeout: "120m"
       - go-test:
           name: "Diagnostics tests"
           test-suite: "./test/minikube"

diff --git a/go.mod b/go.mod
@@ -1,23 +1,71 @@
 module github.com/nuodb/nuodb-helm-charts/v3
 
-go 1.14
+go 1.20
 
 require (
-	cloud.google.com/go v0.61.0 // indirect
 	github.com/Masterminds/semver v1.5.0
-	github.com/aws/aws-sdk-go v1.34.0 // indirect
 	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-cmp v0.5.9
+	github.com/gruntwork-io/terratest v0.43.0
+	github.com/otiai10/copy v1.2.0
+	github.com/stretchr/testify v1.8.1
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.27.2
+	k8s.io/apimachinery v0.27.2
+)
+
+require (
+	github.com/aws/aws-sdk-go v1.44.122 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/go-cmp v0.5.5
-	github.com/gruntwork-io/gruntwork-cli v0.6.1 // indirect
-	github.com/gruntwork-io/terratest v0.28.10
-	github.com/imdario/mergo v0.3.10 // indirect
-	github.com/otiai10/copy v1.2.0
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
-	gopkg.in/yaml.v3 v3.0.0
-	k8s.io/api v0.20.0
-	k8s.io/apimachinery v0.20.0
-	k8s.io/client-go v0.20.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gruntwork-io/go-commons v0.8.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.2.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/urfave/cli v1.22.2 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/oauth2 v0.1.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/client-go v0.27.2 // indirect
+	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
diff --git a/go.sum b/go.sum
diff --git a/scripts/ci/install_deps.sh b/scripts/ci/install_deps.sh
@@ -78,6 +78,10 @@ $ip demo.nuodb.local" | sudo tee -a /etc/hosts
   # get HAProxy for Ingress testing
   helm repo add haproxytech https://haproxytech.github.io/helm-charts
 
+  # enable volume snapshots and CSI
+  minikube addons enable volumesnapshots
+  minikube addons enable csi-hostpath-driver
+
 elif [[ "$REQUIRES_MINISHIFT" == "true" ]]; then
   wget https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz -O /tmp/oc.tar.gz
   tar xzf /tmp/oc.tar.gz -C /tmp --strip-components=1 && chmod +x /tmp/oc && sudo mv /tmp/oc /usr/local/bin

diff --git a/stable/database/README.md b/stable/database/README.md
@@ -212,6 +212,7 @@ The following tables list the configurable parameters of the `database` chart an
 | `persistence.accessModes` | Volume access modes enabled (must match capabilities of the storage class) | `ReadWriteOnce` |
 | `persistence.size` | Amount of disk space allocated for database archive storage | `20Gi` |
 | `persistence.storageClass` | Storage class for volume backing database archive storage | `-` |
+| `persistence.dataSourceRef` | Data Source to initialize the archive volume.  See below.  |  |
 | `configFilesPath` | Directory path where `configFiles.*` are found | `/etc/nuodb/` |
 | `configFiles.*` | See below. | `{}` |
 | `podAnnotations` | Annotations to pass through to the SM an TE pods | `nil` |
@@ -221,6 +222,7 @@ The following tables list the configurable parameters of the `database` chart an
 | `autoImport.credentials` | Authentication for the download of `source` in the form `user`:`password` | '""'|
 | `autoImport.stripLevels` | The number of levels to strip off pathnames when unpacking a TAR file of an archive | `1` |
 | `autoImport.type` | Type of content in `source`. One of `stream` -> exact copy of an archive; or `backupset` -> a NuoDB hotcopy backupset | 'backupset' |
+| `autoImport.backupId` | When specifying a value for `dataSourceRef` (see below), can be used to validate the snapshot| `""` |
 | `autoRestore.*` | Enable and configure the automatic re-initialization of a single archive in a running database - see the options in `autoImport` | `disabled` |
 | `ephemeralVolume.enabled` | Whether to create a generic ephemeral volume rather than emptyDir for any storage that does not outlive the pod | `false` |
 | `ephemeralVolume.size` | The size of the generic ephemeral volume to create | `1Gi` |
@@ -254,6 +256,7 @@ The following tables list the configurable parameters of the `database` chart an
 | `sm.hotCopy.persistence.size` | size of the hotcopy storage PV | `20Gi` |
 | `sm.hotCopy.persistence.accessModes` | access modes for the hotcopy storage PV | `[ ReadWriteOnce ]` |
 | `sm.hotCopy.persistence.size` | size of the hotcopy storage PV | `20Gi` |
+| `sm.hotCopy.journalPath.persistence.dataSourceRef` | Data Source to initialize the journal volume. See below.  |  |
 | `sm.hotCopy.journalBackup.enabled` | Is `journal hotcopy` enabled - true/false | `false` |
 | `sm.hotCopy.journalBackup.journalSchedule` | cron schedule for _JOURNAL_ hotcopy jobs. When journal backup is enabled, an SM will retain each journal file on disk until it is journal hot copied into a backup set. This means that journal hot copy must be executed periodically to prevent SMs from running out of disk space on the journal volume | `?/15 * * * *` |
 | `sm.hotCopy.journalBackup.deadline` | Deadline for a `journal hotcopy` job to start (seconds) | `90` |
@@ -269,6 +272,7 @@ The following tables list the configurable parameters of the `database` chart an
 | `sm.noHotCopy.journalPath.accessModes` | Volume access modes enabled (must match capabilities of the storage class) | `ReadWriteOnce` |
 | `sm.noHotCopy.journalPath.size` | Amount of disk space allocated for SM journal | `20Gi` |
 | `sm.noHotCopy.journalPath.storageClass` | Storage class for SM journal.  This storage class must be pre-configured in the cluster | `-` |
+| `sm.hotCopy.journalPath.persistence.dataSourceRef` | Data Source to initialize the journal volume. See below.  |  |
 | `sm.labels` | Labels given to the SMs started | `{}` |
 | `sm.engineOptions` | Additional NuoDB engine options | `{}` |
 | `sm.resources` | Labels to apply to all resources | `{}` |
@@ -335,6 +339,21 @@ The purpose of this section is to allow customisation of the names of the cluste
 | `balancer` | suffix for the balancer service | .Values.admin.serviceSuffix.balancer |
 | `nodeport` | suffix for the nodePort service | .Values.admin.serviceSuffix.nodeport |
 
+### persistence.dataSourceRef.*
+
+Define a VolumeSnapshot, PersistentVolumeClaim, or other dataSource from which to initialize a database volume.
+
+See [Persistent Volume Doumentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#data-source-references) and [Resource Schema](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#typedobjectreference-v1-core)
+
+When taking a Volume Snapshot, you can drop a file called `backup.txt` in the directory root containing an arbitrary value. That value should be passed in via `database.autoImport.backupId`. Required if restoring a database with the same domain and database names, optional otherwise.
+
+| Key | Description | Default |
+| ----- | ----------- | ------ |
+| `name` | Backup or volume name. The entire dataSource will be omitted if this value is empty | `nil` |
+| `namespace` | Namespace containing the source | `nil` |
+| `kind` | Data source kind | `VolumeSnapshot` |
+| `apiGroup` | APIGroup is the group for the resource. If APIGroup is not specified, the specified Kind must be in the core API group. | `snapshot.storage.k8s.io` |
+
 #### database.legacy
 
 Features in this section have been deprecated but not yet removed.

diff --git a/stable/database/files/nuosm b/stable/database/files/nuosm
@@ -387,6 +387,82 @@ function resolveLatestSource() {
   return 0
 }
 
+#=======================================
+# function - Test that a backup file (the first argument) exists and contains
+# the value of $BACKUP_ID
+#
+function checkBackupId() {
+  local backup_file=$1
+  [ -f $backup_file ] && [ "$(cat $backup_file | tr -d [:space:])" == "$BACKUP_ID" ]
+}
+
+#=======================================
+# function - Looks for a previous archive and journal and imports it
+#
+function loadFromSnapshot() {
+  local recreate_archives="false"
+
+  if [ ! -f "$DB_DIR/state.dat" ]; then
+    local archives=$( find "/var/opt/nuodb/archive" -name info.json )
+    if [ -z "$archives" ] || [ $(echo "$archives" | wc -l) != 1 ]; then 
+      die 1 "Did not find exactly 1 archive: ${archives}"
+    fi
+
+    local archive_path=$( dirname $archives )
+    log "Found snapshot at ${archive_path}"
+
+    if [ -n "$BACKUP_ID" ]; then
+      if ! checkBackupId ${archive_path}/backup.txt ; then
+        die 1 "Incorrect backup id in archive"
+      fi
+    fi
+
+    if [ "$SEPARATE_JOURNAL" == "true" ]; then
+      local relative_archive=$(realpath --relative-base=/var/opt/nuodb/archive $archive_path)
+      local expected_journal="/var/opt/nuodb/journal/${relative_archive}"
+
+      if [ ! -d $expected_journal ]; then
+        die 1 "Did not find a journal snapshot at '$expected_journal'"
+      fi
+
+      if [ -n "$BACKUP_ID" ]; then
+        if ! checkBackupId ${expected_journal}/backup.txt ; then
+          die 1 "Incorrect backup id in journal"
+        fi
+      fi
+
+      rm -rf ${JOURNAL_DIR}
+      mv ${expected_journal} ${JOURNAL_DIR}
+
+    fi
+
+    # Don't move the archive until after we have done error checking.
+    # Once we move the archive, we are going to be in the `archive already exists` branch
+    # when kubernetes retries/restarts the pod.
+    rm -rf ${DB_DIR}
+    mv ${archive_path} ${DB_DIR}
+
+    local recreate_archives="true"
+  else
+    trace "archive already exists, not looking for snapshot archives"
+
+    local backup_file=${DB_DIR}/backup.txt
+    if [ -n "$BACKUP_ID" ] && [ -f  $backup_file ] ; then
+      if [ "$(cat $backup_file | tr -d [:space:])" == "$BACKUP_ID" ] ; then
+        local recreate_archives="true"
+      else
+        log "Archive present but with a wrong backupId. Did the SM crash during a snapshot?"
+      fi
+    fi
+  fi
+
+  if [ $recreate_archives == "true" ]; then
+    log "Removing outdated archive metadata"
+    rm ${DB_DIR}/info.json ${DB_DIR}/backup.txt
+    [ -e ${JOURNAL_DIR}/backup.txt ] && rm ${JOURNAL_DIR}/backup.txt
+  fi
+}
+
 #=============================
 # main routine
 #=============================
@@ -415,6 +491,10 @@ if [ "$SEPARATE_JOURNAL" == "true" ]; then
   log "journal-dir=${JOURNAL_DIR}"
 fi
 
+if [ "$SNAPSHOT_RESTORED" == "true" ]; then
+  loadFromSnapshot
+fi
+
 # Retry the first request against the admin layer to make sure that Raft
 # commands can be committed and fail fast on error
 retry 3 nuocmd get value --key "$liveness_req" 2>&1

diff --git a/stable/database/templates/statefulset.yaml b/stable/database/templates/statefulset.yaml
@@ -173,6 +173,12 @@ spec:
         - { name: OVERWRITE_COPIES,    value: "{{ .Values.database.sm.logPersistence.overwriteBackoff.copies | default "3" }}" }
         - { name: OVERWRITE_WINDOW,    value: "{{ .Values.database.sm.logPersistence.overwriteBackoff.windowMinutes | default "120" }}" }
         - { name: SEPARATE_JOURNAL,    value: "{{- include "defaultfalse" .Values.database.sm.noHotCopy.journalPath.enabled}}" }
+        {{- if not (empty .Values.database.persistence.dataSourceRef) }}
+          {{- if (not (empty .Values.database.persistence.dataSourceRef.name)) }}
+        - { name: SNAPSHOT_RESTORED,   value: "true" }
+          {{- end }}
+        {{- end }}
+        - { name: BACKUP_ID,           value: "{{ .Values.database.autoImport.backupId }}" }
 {{- include "database.env" . | indent 8 }}
     {{- if .Values.admin.tlsKeyStore }}
       {{- if .Values.admin.tlsKeyStore.password }}
@@ -315,6 +321,12 @@ spec:
         matchLabels:
           database: {{ .Values.database.name }}
 {{- end }}
+      {{- if .Values.database.persistence.dataSourceRef }}
+        {{- if .Values.database.persistence.dataSourceRef.name }}
+      dataSourceRef:
+        {{- toYaml .Values.database.persistence.dataSourceRef | nindent 8}}
+        {{- end }}
+      {{- end }}
       resources:
         requests:
           storage: {{ .Values.database.persistence.size }}
@@ -336,6 +348,12 @@ spec:
       storageClassName: {{ .Values.database.sm.noHotCopy.journalPath.persistence.storageClass }}
       {{- end }}
     {{- end }}
+    {{- if .Values.database.sm.noHotCopy.journalPath.persistence.dataSourceRef }}
+      {{- if .Values.database.sm.noHotCopy.journalPath.persistence.dataSourceRef.name }}
+      dataSourceRef:
+      {{- toYaml .Values.database.sm.noHotCopy.journalPath.persistence.dataSourceRef | nindent 8}}
+      {{- end }}
+    {{- end }}
 {{- if .Values.database.isManualVolumeProvisioning }}
       selector:
         matchLabels:
@@ -542,6 +560,12 @@ spec:
         - { name: OVERWRITE_COPIES,  value: "{{ .Values.database.sm.logPersistence.overwriteBackoff.copies | default "3" }}" }
         - { name: OVERWRITE_WINDOW,  value: "{{ .Values.database.sm.logPersistence.overwriteBackoff.windowMinutes | default "120" }}" }
         - { name: SEPARATE_JOURNAL,  value: "{{- include "defaultfalse" .Values.database.sm.hotCopy.journalPath.enabled}}" }
+        {{- if not (empty .Values.database.persistence.dataSourceRef) }}
+          {{- if (not (empty .Values.database.persistence.dataSourceRef.name)) }}
+        - { name: SNAPSHOT_RESTORED,   value: "true" }
+          {{- end }}
+        {{- end }}
+        - { name: BACKUP_ID,         value: "{{ .Values.database.autoImport.backupId }}" }
 {{- include "database.env" . | indent 8 }}
     {{- if .Values.admin.tlsKeyStore }}
       {{- if .Values.admin.tlsKeyStore.password }}
@@ -686,6 +710,12 @@ spec:
         matchLabels:
           database: {{ .Values.database.name }}
 {{- end }}
+      {{- if .Values.database.persistence.dataSourceRef }}
+        {{- if .Values.database.persistence.dataSourceRef.name }}
+      dataSourceRef:
+        {{- toYaml .Values.database.persistence.dataSourceRef | nindent 8}}
+        {{- end }}
+      {{- end }}
       resources:
         requests:
           storage: {{ .Values.database.persistence.size }}
@@ -707,6 +737,12 @@ spec:
       storageClassName: {{ .Values.database.sm.hotCopy.journalPath.persistence.storageClass }}
       {{- end }}
     {{- end }}
+    {{- if .Values.database.sm.hotCopy.journalPath.persistence.dataSourceRef }}
+      {{- if .Values.database.sm.hotCopy.journalPath.persistence.dataSourceRef.name }}
+      dataSourceRef:
+      {{- toYaml .Values.database.sm.hotCopy.journalPath.persistence.dataSourceRef | nindent 8}}
+      {{- end }}
+    {{- end }}
 {{- if .Values.database.isManualVolumeProvisioning }}
       selector:
         matchLabels:

diff --git a/stable/database/values.yaml b/stable/database/values.yaml
@@ -235,6 +235,15 @@ database:
       - ReadWriteOnce
     # storageClass: "-"
 
+    # The data source for the archive PVC
+    # see https://kubernetes.io/docs/concepts/storage/persistent-volumes/#data-source-references
+    # schema https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#typedobjectreference-v1-core
+    dataSourceRef:
+      # name: archive-snapshot
+      # namespace: old-namespace
+      kind: VolumeSnapshot
+      apiGroup: snapshot.storage.k8s.io
+
   ## database-wide options.
   # These are applied using the --database-options on the startup command
   # change these to values appropriate for this database
@@ -265,6 +274,9 @@ database:
     stripLevels: "1"
     type: ""
 
+    # ID to validate dataSourceRef values
+    backupId: ""
+
   # ensure all values here are strings - so quote any purely numeric values
   autoRestore:
     source: ""
@@ -391,6 +403,15 @@ database:
             - ReadWriteOnce
           # storageClass: "-"
 
+          # The data source for journal PVC, only applies to SMs with hotcopy enabled
+          # see https://kubernetes.io/docs/concepts/storage/persistent-volumes/#data-source-references
+          # schema https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#typedobjectreference-v1-core
+          dataSourceRef:
+            # name: journal-snapshot
+            # namespace: old-namespace
+            kind: VolumeSnapshot
+            apiGroup: snapshot.storage.k8s.io
+
       coldStorage:
         credentials: ""
 
@@ -426,6 +447,15 @@ database:
             - ReadWriteOnce
           # storageClass: "-"
 
+          # The data source for journal PVC, only applies to SMs with hotcopy disabled
+          # see https://kubernetes.io/docs/concepts/storage/persistent-volumes/#data-source-references
+          # schema https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#typedobjectreference-v1-core
+          dataSourceRef:
+            # name: journal-snapshot
+            # namespace: old-namespace
+            kind: VolumeSnapshot
+            apiGroup: snapshot.storage.k8s.io
+
     ## resources
     # k8s resource min (request) and max (limit)
     # min is also used for the target maximum memory used by the cache (NuoDB --mem option)