ETCD-478: Use golang etcd client for auto backups instead of cluster-…

…backup.sh
openshift · Oct 16, 2023 · ae0e36c · ae0e36c
1 parent b52b83f
commit ae0e36c
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 27 deletions.
diff --git a/bindata/etcd/cluster-backup-job.yaml b/bindata/etcd/cluster-backup-job.yaml
@@ -37,8 +37,8 @@ spec:
             - |
               #!/bin/sh
               set -exuo pipefail
-              
-              /usr/local/bin/cluster-backup.sh --force ${CLUSTER_BACKUP_PATH}
+
+              cluster-etcd-operator cluster-backup --backup-dir "${CLUSTER_BACKUP_PATH}"
 
           resources:
             requests:
@@ -57,7 +57,12 @@ spec:
               name: static-pod-dir
             - mountPath: /etc/kubernetes/cluster-backup
               name: etc-kubernetes-cluster-backup
+            - mountPath: /var/run/secrets/etcd-client
+              name: etcd-client
+            - mountPath: /var/run/configmaps/etcd-ca
+              name: etcd-ca
       priorityClassName: system-node-critical
+      activeDeadlineSeconds: 900
       nodeSelector:
         node-role.kubernetes.io/master: ""
       restartPolicy: Never

diff --git a/pkg/cmd/backuprestore/etcdclientutils.go b/pkg/cmd/backuprestore/etcdclientutils.go
@@ -27,17 +27,20 @@ func getEtcdClient(endpoints []string) (*clientv3.Client, error) {
 		TrustedCAFile: os.Getenv("ETCDCTL_CACERT"),
 	}
 	tlsConfig, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate tls client config endpoints %v and env %v: %w", endpoints, os.Environ(), err)
+	}
 
 	cfg := &clientv3.Config{
 		DialOptions: dialOptions,
 		Endpoints:   endpoints,
-		DialTimeout: 2 * time.Second,
+		DialTimeout: 10 * time.Second,
 		TLS:         tlsConfig,
 	}
 
 	cli, err := clientv3.New(*cfg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to make etcd client for endpoints %v: %w", endpoints, err)
+		return nil, fmt.Errorf("failed to make etcd client for endpoints %v and env %v: %w", endpoints, os.Environ(), err)
 	}
 	return cli, nil
 }
@@ -64,14 +67,11 @@ func saveSnapshot(cli *clientv3.Client, dbPath string) error {
 	if err := f.Close(); err != nil {
 		return fmt.Errorf("saveSnapshot failed: %w", err)
 	}
-	klog.Info(
-		"fetched snapshot. ",
-		"Time taken: ", time.Since(opBegin),
-	)
+	klog.Infof("fetched snapshot, took: %v", time.Since(opBegin))
 
 	if err := os.Rename(partpath, dbPath); err != nil {
 		return fmt.Errorf("could not rename %s to %s (%v)", partpath, dbPath, err)
 	}
-	klog.Info("saved snapshot to path", dbPath)
+	klog.Infof("saved snapshot to path %s", dbPath)
 	return nil
 }
diff --git a/pkg/operator/backupcontroller/backupcontroller.go b/pkg/operator/backupcontroller/backupcontroller.go
@@ -42,7 +42,6 @@ type BackupController struct {
 	operatorClient        v1helpers.OperatorClient
 	backupsClient         operatorv1alpha1client.EtcdBackupsGetter
 	kubeClient            kubernetes.Interface
-	targetImagePullSpec   string
 	operatorImagePullSpec string
 	featureGateAccessor   featuregates.FeatureGateAccess
 }
@@ -52,7 +51,6 @@ func NewBackupController(
 	backupsClient operatorv1alpha1client.EtcdBackupsGetter,
 	kubeClient kubernetes.Interface,
 	eventRecorder events.Recorder,
-	targetImagePullSpec string,
 	operatorImagePullSpec string,
 	accessor featuregates.FeatureGateAccess,
 	backupInformer factory.Informer,
@@ -61,7 +59,6 @@ func NewBackupController(
 	c := &BackupController{
 		backupsClient:         backupsClient,
 		kubeClient:            kubeClient,
-		targetImagePullSpec:   targetImagePullSpec,
 		operatorImagePullSpec: operatorImagePullSpec,
 		featureGateAccessor:   accessor,
 	}
@@ -155,7 +152,7 @@ func (c *BackupController) sync(ctx context.Context, _ factory.SyncContext) erro
 		return nil
 	}
 
-	err = createBackupJob(ctx, backupsToRun[0], c.targetImagePullSpec, c.operatorImagePullSpec, jobsClient, backupsClient)
+	err = createBackupJob(ctx, backupsToRun[0], c.operatorImagePullSpec, jobsClient, backupsClient)
 	if err != nil {
 		return err
 	}
@@ -387,7 +384,6 @@ func reconcileJobStatus(ctx context.Context,
 
 func createBackupJob(ctx context.Context,
 	backup operatorv1alpha1.EtcdBackup,
-	targetImagePullSpec string,
 	operatorImagePullSpec string,
 	jobClient batchv1client.JobInterface,
 	backupClient operatorv1alpha1client.EtcdBackupInterface) error {
@@ -422,9 +418,13 @@ func createBackupJob(ctx context.Context,
 	}
 
 	job.Spec.Template.Spec.InitContainers[0].Image = operatorImagePullSpec
-	job.Spec.Template.Spec.Containers[0].Image = targetImagePullSpec
+	job.Spec.Template.Spec.Containers[0].Image = operatorImagePullSpec
+
 	job.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{
 		{Name: backupDirEnvName, Value: fmt.Sprintf("%s/%s", recentBackupPath, backupFileName)},
+		{Name: "ETCDCTL_CERT", Value: "/var/run/secrets/etcd-client/tls.crt"},
+		{Name: "ETCDCTL_KEY", Value: "/var/run/secrets/etcd-client/tls.key"},
+		{Name: "ETCDCTL_CACERT", Value: "/var/run/configmaps/etcd-ca/ca-bundle.crt"},
 	}
 
 	injected := false

diff --git a/pkg/operator/backupcontroller/backupcontroller_test.go b/pkg/operator/backupcontroller/backupcontroller_test.go
@@ -36,7 +36,6 @@ func TestSyncLoopHappyPath(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -61,7 +60,6 @@ func TestJobAlreadyRunning(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -90,7 +88,6 @@ func TestJobBackupJobFinished(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -119,7 +116,6 @@ func TestJobWithoutBackupRemovesJob(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -138,7 +134,6 @@ func TestJobCreationHappyPath(t *testing.T) {
 
 	err := createBackupJob(context.Background(),
 		backup,
-		"target-pullspec-image",
 		"operator-pullspec-image",
 		client.BatchV1().Jobs(operatorclient.TargetNamespace),
 		operatorFake.OperatorV1alpha1().EtcdBackups(),
@@ -174,7 +169,6 @@ func TestMultipleBackupsAreSkipped(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -239,7 +233,6 @@ func TestPVCNotFound(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -291,7 +284,6 @@ func TestOwnerRefsPropagate(t *testing.T) {
 	controller := BackupController{
 		backupsClient:         operatorFake.OperatorV1alpha1(),
 		kubeClient:            client,
-		targetImagePullSpec:   "target-pullspec-image",
 		operatorImagePullSpec: "operator-pullspec-image",
 		featureGateAccessor:   backupFeatureGateAccessor,
 	}
@@ -317,7 +309,7 @@ func requireBackupJobCreated(t *testing.T, client *k8sfakeclient.Clientset, back
 	require.Equal(t, operatorclient.TargetNamespace, createdJob.Namespace)
 	require.Equal(t, backup.Name, createdJob.Labels[backupJobLabel])
 	require.Equal(t, "operator-pullspec-image", createdJob.Spec.Template.Spec.InitContainers[0].Image)
-	require.Equal(t, "target-pullspec-image", createdJob.Spec.Template.Spec.Containers[0].Image)
+	require.Equal(t, "operator-pullspec-image", createdJob.Spec.Template.Spec.Containers[0].Image)
 
 	foundVolume := false
 	for _, volume := range createdJob.Spec.Template.Spec.Volumes {

diff --git a/pkg/operator/etcd_assets/bindata.go b/pkg/operator/etcd_assets/bindata.go
diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go
@@ -438,7 +438,6 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 			operatorConfigClientv1Alpha1,
 			kubeClient,
 			controllerContext.EventRecorder,
-			os.Getenv("IMAGE"),
 			os.Getenv("OPERATOR_IMAGE"),
 			featureGateAccessor,
 			etcdBackupInformer,