feat( cluster ): Adds support for recovery.mode=import

charts/cluster/README.md

@@ -217,7 +217,27 @@ refer to  the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat
 | recovery.google.bucket | string | `""` |  |
 | recovery.google.gkeEnvironment | bool | `false` |  |
 | recovery.google.path | string | `"/"` |  |
-| recovery.method | string | `"backup"` | Available recovery methods: * `backup` - Recovers a CNPG cluster from a CNPG backup (PITR supported) Needs to be on the same cluster in the same namespace. * `object_store` - Recovers a CNPG cluster from a barman object store (PITR supported). * `pg_basebackup` - Recovers a CNPG cluster viaa streaming replication protocol. Useful if you want to        migrate databases to CloudNativePG, even from outside Kubernetes. # TODO |
+| recovery.import.databases | list | `[]` | Databases to import |
+| recovery.import.postImportApplicationSQL | list | `[]` | List of SQL queries to be executed as a superuser in the application database right after is imported. To be used with extreme care. Only available in microservice type. |
+| recovery.import.roles | list | `[]` | Roles to import |
+| recovery.import.schemaOnly | bool | `false` | When set to true, only the pre-data and post-data sections of pg_restore are invoked, avoiding data import. |
+| recovery.import.source.database | string | `""` |  |
+| recovery.import.source.host | string | `""` |  |
+| recovery.import.source.passwordSecret.create | bool | `false` | Whether to create a secret for the password |
+| recovery.import.source.passwordSecret.key | string | `"password"` | The key in the secret containing the password |
+| recovery.import.source.passwordSecret.name | string | `""` | Name of the secret containing the password |
+| recovery.import.source.passwordSecret.value | string | `""` | The password value to use when creating the secret |
+| recovery.import.source.port | int | `5432` |  |
+| recovery.import.source.sslCertSecret.key | string | `""` |  |
+| recovery.import.source.sslCertSecret.name | string | `""` |  |
+| recovery.import.source.sslKeySecret.key | string | `""` |  |
+| recovery.import.source.sslKeySecret.name | string | `""` |  |
+| recovery.import.source.sslMode | string | `"verify-full"` |  |
+| recovery.import.source.sslRootCertSecret.key | string | `""` |  |
+| recovery.import.source.sslRootCertSecret.name | string | `""` |  |
+| recovery.import.source.username | string | `""` |  |
+| recovery.import.type | string | `"microservice"` | One of `microservice` or `monolith`. See: https://cloudnative-pg.io/documentation/1.24/database_import/#how-it-works |
+| recovery.method | string | `"backup"` | Available recovery methods: * `backup` - Recovers a CNPG cluster from a CNPG backup (PITR supported) Needs to be on the same cluster in the same namespace. * `object_store` - Recovers a CNPG cluster from a barman object store (PITR supported). * `pg_basebackup` - Recovers a CNPG cluster via streaming replication protocol. Useful if you want to        migrate databases to CloudNativePG, even from outside Kubernetes. * `import` - Import one or more databases from an existing Postgres cluster. |
 | recovery.pgBaseBackup.database | string | `"app"` | Name of the database used by the application. Default: `app`. |
 | recovery.pgBaseBackup.owner | string | `""` | Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch |
 | recovery.pgBaseBackup.secret | string | `""` | Name of the owner of the database in the instance to be used by applications. Defaults to the value of the `database` key. |

charts/cluster/templates/_bootstrap.tpl

@@ -3,7 +3,7 @@
 bootstrap:
   initdb:
     {{- with .Values.cluster.initdb }}
-        {{- with (omit . "postInitApplicationSQL" "owner") }}
+        {{- with (omit . "postInitApplicationSQL" "owner" "import") }}
             {{- . | toYaml | nindent 4 }}
         {{- end }}
     {{- end }}
@@ -43,33 +43,42 @@ bootstrap:
     {{- end }}
 
 externalClusters:
-- name: pgBaseBackupSource
-  connectionParameters:
-    host: {{ .Values.recovery.pgBaseBackup.source.host | quote }}
-    port: {{ .Values.recovery.pgBaseBackup.source.port | quote }}
-    user: {{ .Values.recovery.pgBaseBackup.source.username | quote }}
-    dbname: {{ .Values.recovery.pgBaseBackup.source.database | quote }}
-    sslmode: {{ .Values.recovery.pgBaseBackup.source.sslMode | quote }}
-  {{- if .Values.recovery.pgBaseBackup.source.passwordSecret.name }}
-  password:
-    name: {{ default (printf "%s-pg-basebackup-password" (include "cluster.fullname" .)) .Values.recovery.pgBaseBackup.source.passwordSecret.name }}
-    key: {{ .Values.recovery.pgBaseBackup.source.passwordSecret.key }}
-  {{- end }}
-  {{- if .Values.recovery.pgBaseBackup.source.sslKeySecret.name }}
-  sslKey:
-    name: {{ .Values.recovery.pgBaseBackup.source.sslKeySecret.name }}
-    key: {{ .Values.recovery.pgBaseBackup.source.sslKeySecret.key }}
-  {{- end }}
-  {{- if .Values.recovery.pgBaseBackup.source.sslCertSecret.name }}
-  sslCert:
-    name: {{ .Values.recovery.pgBaseBackup.source.sslCertSecret.name }}
-    key: {{ .Values.recovery.pgBaseBackup.source.sslCertSecret.key }}
-  {{- end }}
-  {{- if .Values.recovery.pgBaseBackup.source.sslRootCertSecret.name }}
-  sslRootCert:
-    name: {{ .Values.recovery.pgBaseBackup.source.sslRootCertSecret.name }}
-    key: {{ .Values.recovery.pgBaseBackup.source.sslRootCertSecret.key }}
-  {{- end }}
+  {{- include "cluster.externalSourceCluster" (list "pgBaseBackupSource" .Values.recovery.pgBaseBackup.source) | nindent 2 }}
+
+{{- else if eq .Values.recovery.method "import" }}
+  initdb:
+    {{- with .Values.cluster.initdb }}
+        {{- with (omit . "owner" "import") }}
+            {{- . | toYaml | nindent 4 }}
+        {{- end }}
+    {{- end }}
+    {{- if .Values.cluster.initdb.owner }}
+    owner: {{ tpl .Values.cluster.initdb.owner . }}
+    {{- end }}
+    import:
+      source:
+        externalCluster: importSource
+      type: {{ .Values.recovery.import.type }}
+      databases: {{ .Values.recovery.import.databases | toJson }}
+      {{ with .Values.recovery.import.roles }}
+      roles: {{ . | toJson }}
+      {{- end }}
+      {{ with .Values.recovery.import.postImportApplicationSQL }}
+      postImportApplicationSQL:
+        {{- . | toYaml | nindent 6 }}
+      {{- end }}
+      schemaOnly: {{ .Values.recovery.import.schemaOnly }}
+      {{ with .Values.recovery.import.pgDumpExtraOptions }}
+      pgDumpExtraOptions:
+        {{- . | toYaml | nindent 6 }}
+      {{- end }}
+      {{ with .Values.recovery.import.pgRestoreExtraOptions }}
+      pgRestoreExtraOptions:
+        {{- . | toYaml | nindent 6 }}
+      {{- end }}
+
+externalClusters:
+  {{- include "cluster.externalSourceCluster" (list "importSource" .Values.recovery.import.source) | nindent 2 }}
 
 {{- else }}
   recovery:

charts/cluster/templates/_external_source_cluster.tpl

@@ -0,0 +1,33 @@
+{{- define "cluster.externalSourceCluster" -}}
+{{- $name := first . -}}
+{{- $config := last . -}}
+- name: {{ first . }}
+  connectionParameters:
+    host: {{ $config.host | quote }}
+    port: {{ $config.port | quote }}
+    user: {{ $config.username | quote }}
+    {{- with $config.database }}
+    dbname: {{ . | quote }}
+    {{- end }}
+    sslmode: {{ $config.sslMode | quote }}
+  {{- if $config.passwordSecret.name }}
+  password:
+    name: {{ $config.passwordSecret.name }}
+    key: {{ $config.passwordSecret.key }}
+  {{- end }}
+  {{- if $config.sslKeySecret.name }}
+  sslKey:
+    name: {{ $config.sslKeySecret.name }}
+    key: {{ $config.sslKeySecret.key }}
+  {{- end }}
+  {{- if $config.sslCertSecret.name }}
+  sslCert:
+    name: {{ $config.sslCertSecret.name }}
+    key: {{ $config.sslCertSecret.key }}
+  {{- end }}
+  {{- if $config.sslRootCertSecret.name }}
+  sslRootCert:
+    name: {{ $config.sslRootCertSecret.name }}
+    key: {{ $config.sslRootCertSecret.key }}
+  {{- end }}
+{{- end }}

charts/cluster/test/postgresql-import/00-source-cluster-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: source-cluster
+status:
+  readyInstances: 1

charts/cluster/test/postgresql-import/00-source-cluster.yaml

@@ -0,0 +1,9 @@
+type: postgresql
+mode: "standalone"
+cluster:
+  instances: 1
+  superuserSecret: source-cluster-superuser
+  storage:
+    size: 256Mi
+backups:
+  enabled: false

charts/cluster/test/postgresql-import/00-source-superuser-password.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: source-cluster-superuser
+type: Opaque
+data:
+  username: "cG9zdGdyZXM="
+  password: "cG9zdGdyZXM="

charts/cluster/test/postgresql-import/01-data_write-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write
+status:
+  succeeded: 1

charts/cluster/test/postgresql-import/01-data_write.yaml

@@ -0,0 +1,33 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: data-write
+        env:
+          - name: DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: source-cluster-superuser
+                key: username
+          - name: DB_PASS
+            valueFrom:
+              secretKeyRef:
+                name: source-cluster-superuser
+                key: password
+          - name: DB_URI
+            value: postgres://$(DB_USER):$(DB_PASS)@source-cluster-rw:5432
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           set -e
+           apk --no-cache add postgresql-client
+           psql "$DB_URI" -c "CREATE DATABASE mygooddb;"
+           psql "$DB_URI/mygooddb" -c "CREATE TABLE mygoodtable (id serial PRIMARY KEY);"
+           psql "$DB_URI/mygooddb" -c "INSERT INTO mygoodtable VALUES (314159265);"
+           psql "$DB_URI/mygooddb" -c "CREATE TABLE mybadtable (id serial PRIMARY KEY);"

charts/cluster/test/postgresql-import/02-import-cluster-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: import-cluster
+status:
+  readyInstances: 2

charts/cluster/test/postgresql-import/02-import-cluster.yaml

@@ -0,0 +1,32 @@
+type: postgresql
+mode: "recovery"
+recovery:
+  method: "import"
+  import:
+    type: "microservice"
+    databases: [ "mygooddb" ]
+    pgDumpExtraOptions:
+      - --table=mygood*
+    source:
+      host: "source-cluster-rw"
+      username: "postgres"
+      passwordSecret:
+        name: source-cluster-superuser
+        key: password
+      sslMode: "require"
+      sslKeySecret:
+        name: source-cluster-replication
+        key: tls.key
+      sslCertSecret:
+        name: source-cluster-replication
+        key: tls.crt
+
+cluster:
+  instances: 2
+  storage:
+    size: 256Mi
+  initdb:
+    database: mygooddb
+
+backups:
+  enabled: false

charts/cluster/test/postgresql-import/03-data_test-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-test
+status:
+  succeeded: 1

charts/cluster/test/postgresql-import/03-data_test.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-test
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: data-test
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: import-cluster-superuser
+                key: uri
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           set -e
+           apk --no-cache add postgresql-client
+           DB_URI=$(echo $DB_URI | sed "s|/\*|/|" )
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mygoodtable$$)' --csv -q 2>/dev/null)" = "t"
+           echo "mygoodtable exist"
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mybadtable$$)' --csv -q 2>/dev/null)" = "f"
+           echo "mybadtable doesn't exist"
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM mygoodtable WHERE id = 314159265)' --csv -q 2>/dev/null)" = "t"
+           echo "mygoodtable contains the desired value"

charts/cluster/test/postgresql-import/04-import-cluster-schema_only-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: import-schemaonly-cluster
+status:
+  readyInstances: 2

charts/cluster/test/postgresql-import/04-import-cluster-schema_only.yaml

@@ -0,0 +1,33 @@
+type: postgresql
+mode: "recovery"
+recovery:
+  method: "import"
+  import:
+    type: "microservice"
+    databases: [ "mygooddb" ]
+    pgRestoreExtraOptions:
+      - --table=mygoodtable
+    schemaOnly: true
+    source:
+      host: "source-cluster-rw"
+      username: "postgres"
+      passwordSecret:
+        name: source-cluster-superuser
+        key: password
+      sslMode: "require"
+      sslKeySecret:
+        name: source-cluster-replication
+        key: tls.key
+      sslCertSecret:
+        name: source-cluster-replication
+        key: tls.crt
+
+cluster:
+  instances: 2
+  storage:
+    size: 256Mi
+  initdb:
+    database: mygooddb
+
+backups:
+  enabled: false

charts/cluster/test/postgresql-import/05-data_test-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-test-schemaonly
+status:
+  succeeded: 1

charts/cluster/test/postgresql-import/05-data_test.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-test-schemaonly
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: data-test
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: import-schemaonly-cluster-superuser
+                key: uri
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           set -e
+           apk --no-cache add postgresql-client
+           DB_URI=$(echo $DB_URI | sed "s|/\*|/|" )
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mygoodtable$$)' --csv -q 2>/dev/null)" = "t"
+           echo "mygoodtable exist"
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $$mybadtable$$)' --csv -q 2>/dev/null)" = "f"
+           echo "mybadtable doesn't exist"
+           test "$(psql "${DB_URI}mygooddb" -t -c 'SELECT COUNT(*) FROM mygoodtable' --csv -q 2>/dev/null)" = "0"
+           echo "mygoodtable is empty"