Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ETCD-681: Add etcd-backup-server container within separate daemonset #1354

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

ETCD-681: Add etcd-backup-server container within separate daemonset #1354

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9b7c66f
ETCD-681: Add etcd-backup-server container within separate daemonset
Elbehery Oct 16, 2024
0870a56
OCPBUGS-43659: fix edit default CR
Elbehery Oct 22, 2024
e67c016
OCPBUGS-43676: fix Backup Spec Empty Schedule
Elbehery Oct 22, 2024
e320c4b
OCPBUGS-43687: fix Empty Backup CR Schedule
Elbehery Oct 22, 2024
3f289fc
ETCD-706: Skip Backup Pods for Etcd Learning Members
Elbehery Dec 15, 2024
ac9675c
use etcd-endpoint CM voting members
Elbehery Jan 1, 2025
1f1b1a7
syntactic sugar
Elbehery Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions pkg/backuphelpers/backupvars.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ import (
prune "github.com/openshift/cluster-etcd-operator/pkg/cmd/prune-backups"
)

const (
defaultNumberBackups = 3
defaultBackupSchedule = "0 0 * * *"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@deads2k ^^

)

type Enqueueable interface {
Enqueue()
}
Expand Down Expand Up @@ -71,6 +76,16 @@ func (b *BackupConfig) ArgList() []string {

if b.spec.Schedule != "" {
args = append(args, fmt.Sprintf("--%s=%s", "schedule", b.spec.Schedule))
} else {
// fix OCPBUGS-43687
args = append(args, fmt.Sprintf("--%s=%s", "schedule", defaultBackupSchedule))
}

// fix OCPBUGS-43676
if b.spec.RetentionPolicy.RetentionType == "" {
args = append(args, fmt.Sprintf("--%s=%s", "type", backupv1alpha1.RetentionTypeNumber))
args = append(args, fmt.Sprintf("--%s=%d", "maxNumberOfBackups", defaultNumberBackups))
return args
}

if b.spec.RetentionPolicy.RetentionType == prune.RetentionTypeNumber {
Expand Down Expand Up @@ -101,6 +116,16 @@ func (b *BackupConfig) ArgString() string {

if b.spec.Schedule != "" {
args = append(args, fmt.Sprintf("- --%s=%s", "schedule", b.spec.Schedule))
} else {
// fix OCPBUGS-43687
args = append(args, fmt.Sprintf("- --%s=%s", "schedule", defaultBackupSchedule))
}

// fix OCPBUGS-43676
if b.spec.RetentionPolicy.RetentionType == "" {
args = append(args, fmt.Sprintf("--%s=%s", "type", backupv1alpha1.RetentionTypeNumber))
args = append(args, fmt.Sprintf("--%s=%d", "maxNumberOfBackups", defaultNumberBackups))
return strings.Join(args, "\n ")
}

if b.spec.RetentionPolicy.RetentionType == prune.RetentionTypeNumber {
Expand Down
13 changes: 10 additions & 3 deletions pkg/backuphelpers/backupvars_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@ func TestBackupConfig_ToArgs(t *testing.T) {
{
"backup spec with timezone and schedule",
createEtcdBackupSpec(timezone, schedule),
" args:\n - --enabled=true\n - --timezone=GMT\n - --schedule=0 */2 * * *",
" args:\n - --enabled=true\n - --timezone=GMT\n - --schedule=0 */2 * * *\n --type=RetentionNumber\n --maxNumberOfBackups=3",
},
{
"backup spec with timezone and empty schedule",
createEtcdBackupSpec(timezone, ""),
" args:\n - --enabled=true\n - --timezone=GMT",
" args:\n - --enabled=true\n - --timezone=GMT\n - --schedule=0 0 * * *\n --type=RetentionNumber\n --maxNumberOfBackups=3",
},
{
"backup spec with empty timezone and schedule",
createEtcdBackupSpec("", schedule),
" args:\n - --enabled=true\n - --schedule=0 */2 * * *",
" args:\n - --enabled=true\n - --schedule=0 */2 * * *\n --type=RetentionNumber\n --maxNumberOfBackups=3",
},
{
"backup spec with timezone and schedule and retention number",
Expand Down Expand Up @@ -78,6 +78,8 @@ func TestBackupConfig_ToArgList(t *testing.T) {
"--enabled=true",
"--timezone=GMT",
"--schedule=0 */2 * * *",
"--type=RetentionNumber",
"--maxNumberOfBackups=3",
},
},
{
Expand All @@ -86,6 +88,9 @@ func TestBackupConfig_ToArgList(t *testing.T) {
[]string{
"--enabled=true",
"--timezone=GMT",
"--schedule=0 0 * * *",
"--type=RetentionNumber",
"--maxNumberOfBackups=3",
},
},
{
Expand All @@ -94,6 +99,8 @@ func TestBackupConfig_ToArgList(t *testing.T) {
[]string{
"--enabled=true",
"--schedule=0 */2 * * *",
"--type=RetentionNumber",
"--maxNumberOfBackups=3",
},
},
{
Expand Down
59 changes: 52 additions & 7 deletions pkg/cmd/backuprestore/backupserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"os/signal"
"path"
"strings"
"syscall"
"time"

Expand All @@ -17,7 +18,13 @@ import (
"github.com/spf13/cobra"
)

const backupVolume = "/var/lib/etcd-auto-backup"
const (
BackupVolume = "/var/lib/etcd-auto-backup"
Elbehery marked this conversation as resolved.
Show resolved Hide resolved
etcdCtlKeyName = "ETCDCTL_KEY"
etcdCtlCertName = "ETCDCTL_CERT"
etcdCtlCACertName = "ETCDCTL_CACERT"
nodeNameEnvVar = "NODE_NAME"
)

var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}

Expand All @@ -29,7 +36,7 @@ type backupRunnerImpl struct{}

func (b backupRunnerImpl) runBackup(backupOpts *backupOptions, pruneOpts *prune.PruneOpts) error {
dateString := time.Now().Format("2006-01-02_150405")
backupOpts.backupDir = path.Join(backupVolume, dateString)
backupOpts.backupDir = path.Join(BackupVolume, dateString)
err := backup(backupOpts)
if err != nil {
return err
Expand All @@ -46,6 +53,7 @@ func (b backupRunnerImpl) runBackup(backupOpts *backupOptions, pruneOpts *prune.
type backupServer struct {
schedule string
timeZone string
nodeName string
enabled bool
cronSchedule cron.Schedule
backupOptions
Expand All @@ -57,15 +65,14 @@ func NewBackupServer(ctx context.Context) *cobra.Command {
backupOptions: backupOptions{errOut: os.Stderr},
PruneOpts: prune.PruneOpts{
RetentionType: "None",
BackupPath: backupVolume,
BackupPath: BackupVolume,
},
}

cmd := &cobra.Command{
Use: "backup-server",
Short: "Backs up a snapshot of etcd database and static pod resources without config",
Run: func(cmd *cobra.Command, args []string) {

if err := backupSrv.Validate(); err != nil {
klog.Fatal(err)
}
Expand Down Expand Up @@ -95,13 +102,22 @@ func (b *backupServer) Validate() error {
return nil
}

if err := b.validateNameNode(); err != nil {
return err
}

if err := b.constructEnvVars(); err != nil {
klog.Infof("error constructing envVars: [%v]", err)
return err
}

cronSchedule, err := cron.ParseStandard(b.schedule)
if err != nil {
return fmt.Errorf("error parsing backup schedule %v: %w", b.schedule, err)
}
b.cronSchedule = cronSchedule

b.backupOptions.backupDir = backupVolume
b.backupOptions.backupDir = BackupVolume
err = b.backupOptions.Validate()
if err != nil {
return fmt.Errorf("error validating backup %v: %w", b.backupOptions, err)
Expand All @@ -111,7 +127,6 @@ func (b *backupServer) Validate() error {
if err != nil {
return fmt.Errorf("error validating prune args %v: %w", b.PruneOpts, err)
}

return nil
}

Expand All @@ -136,7 +151,6 @@ func (b *backupServer) Run(ctx context.Context) error {
func (b *backupServer) scheduleBackup(ctx context.Context, bck backupRunner) error {
ticker := time.NewTicker(time.Until(b.cronSchedule.Next(time.Now())))
defer ticker.Stop()

for {
select {
case <-ticker.C:
Expand All @@ -151,3 +165,34 @@ func (b *backupServer) scheduleBackup(ctx context.Context, bck backupRunner) err
}
}
}

func (b *backupServer) validateNameNode() error {
nodeNameEnv := os.Getenv(nodeNameEnvVar)
if len(nodeNameEnv) == 0 {
return fmt.Errorf("[%v] environment variable is empty", nodeNameEnvVar)
}
b.nodeName = nodeNameEnv
return nil
}

func (b *backupServer) constructEnvVars() error {
etcdCtlKeyVal := strings.Replace("/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.key", nodeNameEnvVar, b.nodeName, -1)
err := os.Setenv(etcdCtlKeyName, etcdCtlKeyVal)
if err != nil {
return fmt.Errorf("error exporting [%v]: val is [%v]: %w", etcdCtlKeyName, etcdCtlKeyVal, err)
}

etcdCtlCertVal := strings.Replace("/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.crt", nodeNameEnvVar, b.nodeName, -1)
err = os.Setenv(etcdCtlCertName, etcdCtlCertVal)
if err != nil {
return fmt.Errorf("error writing [%v]: val is [%v]: %w", etcdCtlCertName, etcdCtlCertVal, err)
}

etcdCtlCACertVal := "/etc/kubernetes/static-pod-certs/configmaps/etcd-all-bundles/server-ca-bundle.crt"
err = os.Setenv(etcdCtlCACertName, etcdCtlCACertVal)
if err != nil {
return fmt.Errorf("error writing [%v]: val is [%v]: %w", etcdCtlCACertName, etcdCtlCACertVal, err)
}
Elbehery marked this conversation as resolved.
Show resolved Hide resolved

return nil
}
Loading