From 797f5176049cf6bfd18becd4fabb67f1d427a765 Mon Sep 17 00:00:00 2001 From: adriantpaez Date: Tue, 3 Oct 2023 21:20:54 +0000 Subject: [PATCH 1/5] feat: improve backup ls command --- cli/backup_ls.go | 19 +++++- cli/backup_ls_test.go | 104 +++++++++++++++++++++++++++++++ e2e/backup_test.go | 20 +++--- e2e/checks.go | 32 ++-------- internal/backup/manager.go | 72 ++++++++++++++-------- internal/data/backup.go | 112 ++++++++++++++++++++++------------ internal/data/backup_test.go | 105 +++++++++++++++++++++++++++++++ internal/data/datadir.go | 96 ++++++++++++++--------------- internal/data/datadir_test.go | 28 +++++---- internal/utils/tar.go | 55 +++++++++++++++++ pkg/daemon/backup.go | 4 -- pkg/daemon/daemon.go | 6 +- pkg/daemon/egn_daemon.go | 18 ++++-- 13 files changed, 498 insertions(+), 173 deletions(-) create mode 100644 cli/backup_ls_test.go diff --git a/cli/backup_ls.go b/cli/backup_ls.go index 871912d6..5edaddbb 100644 --- a/cli/backup_ls.go +++ b/cli/backup_ls.go @@ -30,23 +30,38 @@ func BackupLsCmd(d daemon.Daemon) *cobra.Command { func printBackupTable(backups []daemon.BackupInfo, out io.Writer) { w := tabwriter.NewWriter(out, 0, 0, 4, ' ', 0) - fmt.Fprintln(w, "AVS Instance ID\tTIMESTAMP\tSIZE\t") + fmt.Fprintln(w, "ID\tAVS Instance ID\tVERSION\tCOMMIT\tTIMESTAMP\tSIZE\tURL\t") for _, b := range backups { fmt.Fprintln(w, backupTableItem{ + id: b.Id, instance: b.Instance, timestamp: b.Timestamp.Format(time.DateTime), size: datasize.Size(b.SizeBytes).String(), + version: b.Version, + commit: b.Commit, + url: b.Url, }) } w.Flush() } type backupTableItem struct { + id string instance string timestamp string size string + version string + commit string + url string +} + +func minifiedId(id string) string { + if len(id) > 8 { + return id[:8] + } + return id } func (b backupTableItem) String() string { - return fmt.Sprintf("%s\t%s\t%s\t", b.instance, b.timestamp, b.size) + return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t", minifiedId(b.id), b.instance, b.version, b.commit, b.timestamp, b.size, b.url) } diff --git a/cli/backup_ls_test.go b/cli/backup_ls_test.go new file mode 100644 index 00000000..3f4d15af --- /dev/null +++ b/cli/backup_ls_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "bytes" + "testing" + "time" + + "github.com/NethermindEth/eigenlayer/cli/mocks" + "github.com/NethermindEth/eigenlayer/pkg/daemon" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackupLs(t *testing.T) { + tc := []struct { + name string + err error + stdErr []byte + stdOut []byte + mocker func(d *mocks.MockDaemon) + }{ + { + name: "no backups", + err: nil, + stdErr: nil, + stdOut: []byte("ID AVS Instance ID VERSION COMMIT TIMESTAMP SIZE URL \n"), + mocker: func(d *mocks.MockDaemon) { + d.EXPECT().BackupList().Return([]daemon.BackupInfo{}, nil) + }, + }, + { + name: "with backups", + err: nil, + stdErr: nil, + stdOut: []byte( + "ID AVS Instance ID VERSION COMMIT TIMESTAMP SIZE URL \n" + + "33de69fe mock-avs-default v5.5.0 a3406616b848164358fdd24465b8eecda5f5ae34 2023-10-03 21:18:36 10KiB https://github.com/NethermindEth/mock-avs-pkg \n" + + "7ba32f63 mock-avs-second v5.5.1 d5af645fffb93e8263b099082a4f512e1917d0af 2023-10-04 07:12:19 10KiB https://github.com/NethermindEth/mock-avs-pkg \n", + ), + mocker: func(d *mocks.MockDaemon) { + d.EXPECT().BackupList().Return([]daemon.BackupInfo{ + { + Id: "33de69fe9225b95c8fb909cb418e5102970c8d73", + Instance: "mock-avs-default", + Version: "v5.5.0", + Commit: "a3406616b848164358fdd24465b8eecda5f5ae34", + Timestamp: time.Date(2023, 10, 3, 21, 18, 36, 0, time.UTC), + SizeBytes: 10240, + Url: "https://github.com/NethermindEth/mock-avs-pkg", + }, + { + Id: "7ba32f630af2cede1388b5712d6ef3ac63175bae", + Instance: "mock-avs-second", + Version: "v5.5.1", + Commit: "d5af645fffb93e8263b099082a4f512e1917d0af", + Timestamp: time.Date(2023, 10, 4, 7, 12, 19, 0, time.UTC), + SizeBytes: 10240, + Url: "https://github.com/NethermindEth/mock-avs-pkg", + }, + }, nil) + }, + }, + { + name: "error", + err: assert.AnError, + stdErr: []byte("Error: " + assert.AnError.Error() + "\n"), + stdOut: []byte{}, + mocker: func(d *mocks.MockDaemon) { + d.EXPECT().BackupList().Return(nil, assert.AnError) + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + d := mocks.NewMockDaemon(ctrl) + + tt.mocker(d) + + var ( + stdOut bytes.Buffer + stdErr bytes.Buffer + ) + + backupLsCmd := BackupLsCmd(d) + backupLsCmd.SetOut(&stdOut) + backupLsCmd.SetErr(&stdErr) + err := backupLsCmd.Execute() + + if tt.err != nil { + require.Error(t, err) + assert.EqualError(t, err, tt.err.Error()) + assert.Equal(t, tt.stdErr, stdErr.Bytes()) + } else { + require.NoError(t, err) + assert.Equal(t, tt.stdOut, stdOut.Bytes()) + } + }) + } +} diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 73a0e08e..6ea7a08a 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -3,24 +3,23 @@ package e2e import ( "regexp" "testing" - "time" "github.com/NethermindEth/eigenlayer/internal/common" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBackupInstance(t *testing.T) { // Test context var ( + output []byte backupErr error - start time.Time ) // Build test case e2eTest := newE2ETestCase( t, // Arrange func(t *testing.T, egnPath string) error { - start = time.Now() err := buildOptionReturnerImageLatest(t) if err != nil { return err @@ -30,12 +29,18 @@ func TestBackupInstance(t *testing.T) { }, // Act func(t *testing.T, egnPath string) { - backupErr = runCommand(t, egnPath, "backup", "mock-avs-default") + output, backupErr = runCommandOutput(t, egnPath, "backup", "mock-avs-default") }, // Assert func(t *testing.T) { assert.NoError(t, backupErr, "backup command should succeed") - checkBackupExist(t, "mock-avs-default", start, time.Now()) + + r := regexp.MustCompile(`.*"Backup created with id: (?P[a-f0-9]+)".*`) + match := r.FindSubmatch(output) + require.Len(t, match, 2, "backup command output should match regex") + instanceId := string(match[1]) + + checkBackupExist(t, instanceId) }, ) // Run test case @@ -72,9 +77,8 @@ func TestBackupList(t *testing.T) { func(t *testing.T) { t.Log(string(out)) assert.NoError(t, backupErr, "backup ls command should succeed") - assert.Regexp(t, regexp.MustCompile( - `AVS Instance ID TIMESTAMP SIZE -mock-avs-default .* 9KiB`), + assert.Regexp(t, regexp.MustCompile(`ID\s+AVS Instance ID\s+VERSION\s+COMMIT\s+TIMESTAMP\s+SIZE\s+URL\s+`+ + `[a-f0-9]{8}\s+mock-avs-default\s+v5\.5\.1\s+d5af645fffb93e8263b099082a4f512e1917d0af\s+.*\s+10KiB\s+https://github.com/NethermindEth/mock-avs-pkg\s+`), string(out)) }, ) diff --git a/e2e/checks.go b/e2e/checks.go index 42dcd0a8..2b185489 100644 --- a/e2e/checks.go +++ b/e2e/checks.go @@ -4,13 +4,11 @@ import ( "context" "fmt" "net/url" - "os" "path/filepath" "slices" "testing" "time" - "github.com/NethermindEth/eigenlayer/internal/data" "github.com/NethermindEth/eigenlayer/internal/env" "github.com/NethermindEth/eigenlayer/pkg/monitoring" "github.com/cenkalti/backoff" @@ -367,32 +365,10 @@ func checkEnvTargets(t *testing.T, instanceId string, targets ...string) { } } -// checkBackupExist checks that a backup exists for the given instanceId between -// the given times a and b, inclusive. -func checkBackupExist(t *testing.T, instanceId string, a, b time.Time) { +// TODO add doc +func checkBackupExist(t *testing.T, backupId string) { dataDir, err := dataDirPath() require.NoError(t, err) - backupsDir := filepath.Join(dataDir, "backup") - - dirFiles, err := os.ReadDir(backupsDir) - require.NoError(t, err) - - var found bool - for _, f := range dirFiles { - if f.IsDir() { - continue - } - i, timestamp, err := data.ParseBackupName(filepath.Base(f.Name())) - if err != nil { - continue - } - if i == instanceId && - ((timestamp.After(a)) && timestamp.Before(b) || - timestamp.Equal(a) || - timestamp.Equal(b)) { - found = true - break - } - } - assert.True(t, found) + backupPath := filepath.Join(dataDir, "backup", backupId+".tar") + assert.FileExists(t, backupPath) } diff --git a/internal/backup/manager.go b/internal/backup/manager.go index 689385cc..f1b34136 100644 --- a/internal/backup/manager.go +++ b/internal/backup/manager.go @@ -59,11 +59,15 @@ func (b *BackupManager) BackupInstance(instanceId string) (string, error) { return "", err } - backupId := data.BackupId{ + backup := &data.Backup{ InstanceId: instanceId, Timestamp: time.Now(), + Version: instance.Version, + Commit: instance.Commit, + Url: instance.URL, } - backup, err := b.dataDir.InitBackup(backupId) + + err = b.dataDir.InitBackup(backup) if err != nil { return "", err } @@ -82,35 +86,18 @@ func (b *BackupManager) BackupInstance(instanceId string) (string, error) { return "", err } - return backup.BackupId.String(), nil -} - -// BackupList returns a list of all backups. -func (b *BackupManager) BackupList() ([]BackupInfo, error) { - // Get the list of backup paths from the data dir - backups, err := b.dataDir.BackupList() + // Add timestamp + err = b.addTimestamp(backup) if err != nil { - return nil, err - } - // Build backup info for each backup - var backupsInfo []BackupInfo - for _, b := range backups { - size, err := b.Size() - if err != nil { - return nil, err - } - backupsInfo = append(backupsInfo, BackupInfo{ - Instance: b.InstanceId(), - Timestamp: b.Timestamp(), - SizeBytes: size, - }) + return "", err } - return backupsInfo, nil + + return backup.Id(), nil } -func (b *BackupManager) backupInstanceData(instanceId string, backup *data.Backup) (err error) { +func (b *BackupManager) backupInstanceData(instanceId string, backup *data.Backup) error { log.Info("Backing up instance data...") - backupPath := backup.Path() + backupPath := b.dataDir.BackupPath(backup.Id()) tarFile, err := b.fs.OpenFile(backupPath, os.O_RDWR, 0o644) if err != nil { return err @@ -128,11 +115,42 @@ func (b *BackupManager) backupInstanceData(instanceId string, backup *data.Backu return utils.TarAddDir(instancePath, filepath.Join("data"), tarFile) } +func (b *BackupManager) addTimestamp(backup *data.Backup) error { + log.Infof("Adding timestamp %s...", backup.Timestamp.Format(time.DateTime)) + backupPath := b.dataDir.BackupPath(backup.Id()) + tarFile, err := b.fs.OpenFile(backupPath, os.O_RDWR, 0o644) + if err != nil { + return err + } + defer tarFile.Close() + + err = utils.TarPrepareToAppend(tarFile) + if err != nil { + return err + } + + timestampTmp, err := afero.TempFile(b.fs, afero.GetTempDir(b.fs, ""), "backup-timestamp-*") + if err != nil { + return err + } + defer timestampTmp.Close() + defer b.fs.Remove(timestampTmp.Name()) + + _, err = timestampTmp.WriteString(fmt.Sprintf("%d", backup.Timestamp.Unix())) + if err != nil { + return err + } + + return utils.TarAddFile(timestampTmp.Name(), "timestamp", tarFile) +} + func (b *BackupManager) backupInstanceServiceVolumes(service types.ServiceConfig, backup *data.Backup) (err error) { if len(service.Volumes) == 0 { return nil } log.Infof("Backing up %d volumes from service \"%s\"...", len(service.Volumes), service.Name) + backupPath := b.dataDir.BackupPath(backup.Id()) + volumes := make([]string, 0, len(service.Volumes)) for _, v := range service.Volumes { volumes = append(volumes, v.Target) @@ -160,7 +178,7 @@ func (b *BackupManager) backupInstanceServiceVolumes(service types.ServiceConfig }, { Type: docker.VolumeTypeBind, - Source: backup.Path(), + Source: backupPath, Target: "/backup.tar", }, }, diff --git a/internal/data/backup.go b/internal/data/backup.go index a5c7f92a..db645a9c 100644 --- a/internal/data/backup.go +++ b/internal/data/backup.go @@ -1,73 +1,109 @@ package data import ( + "crypto/sha1" + "encoding/hex" + "encoding/json" "fmt" + "os" "path/filepath" "regexp" "strconv" "time" + "github.com/NethermindEth/eigenlayer/internal/utils" "github.com/spf13/afero" ) -type BackupId struct { +var backupFileNameRegex = regexp.MustCompile(`^(?P.*)-(?P[0-9]+)\.tar$`) + +type Backup struct { + id string InstanceId string Timestamp time.Time + Version string + Commit string + Url string } -func (b *BackupId) String() string { - return fmt.Sprintf("%s-%d", b.InstanceId, b.Timestamp.Unix()) -} - -type Backup struct { - BackupId - path string - fs afero.Fs +func (b *Backup) Id() string { + if b.id == "" { + h := sha1.Sum([]byte(fmt.Sprintf("%s-%d-%s-%s", b.InstanceId, b.Timestamp.Unix(), b.Version, b.Commit))) + b.id = hex.EncodeToString(h[:]) + } + return b.id } -// NewBackup creates a new Backup instance from the given path. -func NewBackup(fs afero.Fs, path string) (*Backup, error) { - backupFileName := filepath.Base(path) - instanceId, timestamp, err := ParseBackupName(backupFileName) +// BackupFromTar loads a backup information from a tar file. +func BackupFromTar(fs afero.Fs, src string) (*Backup, error) { + // Check if file exists + ok, err := afero.Exists(fs, src) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%w: %s", os.ErrNotExist, src) + } + // Check file name extension + if ext := filepath.Ext(src); ext != ".tar" { + return nil, fmt.Errorf("%w: %s", ErrInvalidBackupName, src) + } + // Load state.json from tar + instance, err := loadBackupTarStateJson(fs, src) + if err != nil { + return nil, err + } + // Load timestamp + timestamp, err := loadBackupTarTimestamp(fs, src) if err != nil { return nil, err } return &Backup{ - BackupId: BackupId{ - InstanceId: instanceId, - Timestamp: timestamp, - }, - path: path, - fs: fs, + InstanceId: instance.ID(), + Timestamp: timestamp, + Version: instance.Version, + Commit: instance.Commit, + Url: instance.URL, }, nil } -// Path returns the path of the backup. -func (b *Backup) Path() string { - return b.path -} - -// InstanceId returns the instance ID of the backup. -func (b *Backup) InstanceId() string { - return b.BackupId.InstanceId -} - -// Timestamp returns the timestamp of the backup. -func (b *Backup) Timestamp() time.Time { - return b.BackupId.Timestamp +// loadStateJsonFromTar loads the state.json file from a tar file.F +func loadBackupTarStateJson(fs afero.Fs, tarPath string) (*Instance, error) { + // Open tar file + tarFile, err := fs.OpenFile(tarPath, os.O_RDONLY, 0o644) + if err != nil { + return nil, err + } + defer tarFile.Close() + // Load state.json + stateData, err := utils.TarReadFile("data/state.json", tarFile) + if err != nil { + return nil, err + } + var instance Instance + return &instance, json.Unmarshal(stateData, &instance) } -// Size returns the size of the backup in bytes. -func (b *Backup) Size() (uint64, error) { - bStat, err := b.fs.Stat(b.path) +func loadBackupTarTimestamp(fs afero.Fs, tarPath string) (time.Time, error) { + // Open file + tarFile, err := fs.OpenFile(tarPath, os.O_RDONLY, 0o644) + if err != nil { + return time.Time{}, err + } + defer tarFile.Close() + // Load timestamp + timestampData, err := utils.TarReadFile("timestamp", tarFile) + if err != nil { + return time.Time{}, err + } + timestampInt, err := strconv.ParseInt(string(timestampData), 10, 64) if err != nil { - return 0, err + return time.Time{}, err } - return uint64(bStat.Size()), nil + return time.Unix(timestampInt, 0), nil } func ParseBackupName(backupName string) (instanceId string, timestamp time.Time, err error) { - backupFileNameRegex := regexp.MustCompile(`^(?P.*)-(?P[0-9]+)\.tar$`) match := backupFileNameRegex.FindStringSubmatch(backupName) if len(match) != 3 { return "", time.Time{}, fmt.Errorf("%w: %s", ErrInvalidBackupName, backupName) diff --git a/internal/data/backup_test.go b/internal/data/backup_test.go index 17d8d4d0..2d5c606d 100644 --- a/internal/data/backup_test.go +++ b/internal/data/backup_test.go @@ -1,13 +1,27 @@ package data import ( + "archive/tar" + "strconv" "testing" "time" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestBackupId(t *testing.T) { + b := Backup{ + InstanceId: "mock-avs-default", + Timestamp: time.Unix(1696367916, 0), + Version: "v5.5.0", + Commit: "a3406616b848164358fdd24465b8eecda5f5ae34", + Url: "https://github.com/NethermindEth/mock-avs-pkg", + } + assert.Equal(t, b.Id(), "33de69fe9225b95c8fb909cb418e5102970c8d73") +} + func TestParseBackupName(t *testing.T) { tc := []struct { name string @@ -58,3 +72,94 @@ func TestParseBackupName(t *testing.T) { }) } } + +func TestBackupFromTar(t *testing.T) { + fs := afero.NewOsFs() + + // Create backup tar + backupTar, err := afero.TempFile(fs, "", "backup-*.tar") + require.NoError(t, err) + defer backupTar.Close() + + // Create state.json + stateFile, err := afero.TempFile(fs, "", "state-*.json") + require.NoError(t, err) + defer stateFile.Close() + stateData := []byte( + `{ + "name": "mock-avs", + "url": "https://github.com/NethermindEth/mock-avs-pkg", + "version": "v5.5.0", + "spec_version": "v0.1.0", + "commit": "a3406616b848164358fdd24465b8eecda5f5ae34", + "profile": "option-returner", + "tag": "default", + "monitoring": { + "targets": [ + { + "service": "main-service", + "port": "8080", + "path": "/metrics" + } + ] + }, + "api": { + "service": "main-service", + "port": "8080" + }, + "plugin": { + "image": "mock-avs-plugin:v0.1.0" + } + }`, + ) + _, err = stateFile.Write(stateData) + require.NoError(t, err) + + // Create timestamp + timestamp := time.Unix(1696367916, 0) + timestampData := []byte(strconv.FormatInt(timestamp.Unix(), 10)) + timestampFile, err := afero.TempFile(fs, "", "timestamp-*") + require.NoError(t, err) + defer timestampFile.Close() + timestampFile.Write(timestampData) + + // Build backup tar + tarWriter := tar.NewWriter(backupTar) + defer tarWriter.Close() + // Add state.json + stateFileInfo, err := stateFile.Stat() + require.NoError(t, err) + h, err := tar.FileInfoHeader(stateFileInfo, "") + require.NoError(t, err) + h.Name = "data/state.json" + err = tarWriter.WriteHeader(h) + require.NoError(t, err) + _, err = tarWriter.Write(stateData) + require.NoError(t, err) + // Add timestamp + timestampFileInfo, err := timestampFile.Stat() + require.NoError(t, err) + h, err = tar.FileInfoHeader(timestampFileInfo, "") + require.NoError(t, err) + h.Name = "timestamp" + err = tarWriter.WriteHeader(h) + require.NoError(t, err) + _, err = tarWriter.Write(timestampData) + require.NoError(t, err) + // Close tar writer + tarWriter.Close() + + // Check backup from tar + b, err := BackupFromTar(fs, backupTar.Name()) + require.NoError(t, err) + require.NotNil(t, b) + assert.Equal(t, + Backup{ + InstanceId: "mock-avs-default", + Timestamp: timestamp, + Version: "v5.5.0", + Commit: "a3406616b848164358fdd24465b8eecda5f5ae34", + Url: "https://github.com/NethermindEth/mock-avs-pkg", + }, + *b) +} diff --git a/internal/data/datadir.go b/internal/data/datadir.go index 8c481e15..915f8a56 100644 --- a/internal/data/datadir.go +++ b/internal/data/datadir.go @@ -164,82 +164,82 @@ func (d *DataDir) TempPath(id string) (string, error) { return tempPath, nil } -func (d *DataDir) InitBackup(backupId BackupId) (*Backup, error) { - if err := d.initBackupDir(); err != nil { - return nil, err - } - return d.initBackup(backupId) -} - -// BackupList returns the list of paths to all the backups. Only .tar files are -// returned. -func (d *DataDir) BackupList() ([]*Backup, error) { - // Init backup dir - if err := d.initBackupDir(); err != nil { +// BackupList returns the list of paths to all the backups. +func (d *DataDir) BackupList() ([]Backup, error) { + err := d.initBackupDir() + if err != nil { return nil, err } - // Get all files in backup dir - dirItems, err := afero.ReadDir(d.fs, filepath.Join(d.path, backupDir)) + backupFiles, err := afero.ReadDir(d.fs, d.backupsDir()) if err != nil { return nil, err } - // Filter .tar files - var backups []*Backup - for _, dirItem := range dirItems { - if dirItem.IsDir() { - continue - } - if filepath.Ext(dirItem.Name()) == ".tar" { - b, err := NewBackup(d.fs, filepath.Join(d.path, backupDir, dirItem.Name())) + + var backups []Backup + for _, backupFile := range backupFiles { + if !backupFile.IsDir() && filepath.Ext(backupFile.Name()) == ".tar" { + b, err := BackupFromTar(d.fs, filepath.Join(d.backupsDir(), backupFile.Name())) if err != nil { return nil, err } - backups = append(backups, b) + backups = append(backups, *b) } } return backups, nil } -func (d *DataDir) initBackup(backupId BackupId) (*Backup, error) { - backupPath, err := d.backupPath(backupId) +// BackupSize returns the size in bytes of the backup with the given id. +func (d *DataDir) BackupSize(backupId string) (int64, error) { + backupStat, err := d.fs.Stat(d.BackupPath(backupId)) if err != nil { - return nil, err - } - - ok, err := d.hasBackup(backupId) - if err != nil { - return nil, err - } - if ok { - return nil, fmt.Errorf("%w: %s", ErrBackupAlreadyExists, backupId) + return -1, err } + return backupStat.Size(), nil +} - err = utils.TarInit(d.fs, backupPath) +// HasBackup returns true if the backup with the given id exists. +func (d *DataDir) HasBackup(backupId string) (bool, error) { + _, err := d.fs.Stat(d.BackupPath(backupId)) if err != nil { - return nil, err + if os.IsNotExist(err) { + return false, nil + } + return false, err } + return true, nil +} - return &Backup{ - BackupId: backupId, - path: backupPath, - fs: d.fs, - }, nil +// BackupPath returns the path to the backup with the given id. +func (d *DataDir) BackupPath(backupId string) string { + return filepath.Join(d.path, backupDir, backupId+".tar") } -func (d *DataDir) hasBackup(backupId BackupId) (bool, error) { - backupPath, err := d.backupPath(backupId) +// InitBackup initialized a new backup. If a backup with the same id already +// exists, an ErrBackupAlreadyExists error is returned. +func (d *DataDir) InitBackup(b *Backup) error { + // Check if backup already exists + exists, err := d.HasBackup(b.Id()) if err != nil { - return false, err + return err + } + if exists { + return fmt.Errorf("%w: %s", ErrBackupAlreadyExists, b.Id()) + } + // Create backup directory if it does not exist + err = d.initBackupDir() + if err != nil { + return err } - return afero.Exists(d.fs, backupPath) + // Initialize backup tar file + return utils.TarInit(d.fs, d.BackupPath(b.Id())) } -func (d *DataDir) backupPath(backupId BackupId) (string, error) { - return filepath.Join(d.path, backupDir, backupId.String()+".tar"), nil +func (d *DataDir) backupsDir() string { + return filepath.Join(d.path, backupDir) } func (d *DataDir) initBackupDir() error { - backupDirPath := filepath.Join(d.path, backupDir) + backupDirPath := d.backupsDir() ok, err := afero.DirExists(d.fs, backupDirPath) if err != nil { return err diff --git a/internal/data/datadir_test.go b/internal/data/datadir_test.go index da684ba1..94a9f590 100644 --- a/internal/data/datadir_test.go +++ b/internal/data/datadir_test.go @@ -760,10 +760,13 @@ func TestDataDir_initBackupDir(t *testing.T) { } } -func TestDataDir_hasBackup(t *testing.T) { - backupId := BackupId{ +func TestDataDir_HasBackup(t *testing.T) { + backup := Backup{ InstanceId: "mock-avs-default", - Timestamp: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Timestamp: time.Unix(1696340865, 0), + Version: common.MockAvsPkg.Version(), + Commit: common.MockAvsPkg.CommitHash(), + Url: common.MockAvsPkg.Repo(), } tc := []struct { name string @@ -808,7 +811,7 @@ func TestDataDir_hasBackup(t *testing.T) { testDir := t.TempDir() err := fs.MkdirAll(filepath.Join(testDir, backupDir), 0o755) require.NoError(t, err) - file, err := fs.Create(filepath.Join(testDir, backupDir, "mock-avs-default-1672531200.tar")) + file, err := fs.Create(filepath.Join(testDir, backupDir, backup.Id()+".tar")) require.NoError(t, err) require.NoError(t, file.Close()) return &DataDir{ @@ -821,7 +824,7 @@ func TestDataDir_hasBackup(t *testing.T) { for _, tt := range tc { t.Run(tt.name, func(t *testing.T) { d := tt.setup() - ok, err := d.hasBackup(backupId) + ok, err := d.HasBackup(backup.Id()) if tt.err != nil { require.Error(t, err) assert.False(t, ok) @@ -834,9 +837,12 @@ func TestDataDir_hasBackup(t *testing.T) { } func TestDataDir_InitBackup(t *testing.T) { - backupId := BackupId{ + backup := Backup{ InstanceId: "mock-avs-default", - Timestamp: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Timestamp: time.Unix(1696340865, 0), + Version: common.MockAvsPkg.Version(), + Commit: common.MockAvsPkg.CommitHash(), + Url: common.MockAvsPkg.Repo(), } tc := []struct { name string @@ -877,7 +883,7 @@ func TestDataDir_InitBackup(t *testing.T) { testDir := t.TempDir() err := fs.MkdirAll(filepath.Join(testDir, backupDir), 0o755) require.NoError(t, err) - file, err := fs.Create(filepath.Join(testDir, backupDir, "mock-avs-default-1672531200.tar")) + file, err := fs.Create(filepath.Join(testDir, backupDir, backup.Id()+".tar")) require.NoError(t, err) require.NoError(t, file.Close()) return &DataDir{ @@ -890,14 +896,12 @@ func TestDataDir_InitBackup(t *testing.T) { for _, tt := range tc { t.Run(tt.name, func(t *testing.T) { d := tt.setup() - b, err := d.initBackup(backupId) + err := d.InitBackup(&backup) if tt.err != nil { assert.ErrorIs(t, err, tt.err) } else { require.NoError(t, err) - require.NotNil(t, b) - assert.Equal(t, backupId, b.BackupId) - bStat, err := d.fs.Stat(b.path) + bStat, err := d.fs.Stat(d.BackupPath(backup.Id())) require.NoError(t, err) require.Equal(t, bStat.Mode(), os.FileMode(0o644)) require.Equal(t, bStat.Size(), int64(1024)) diff --git a/internal/utils/tar.go b/internal/utils/tar.go index 8d7bd067..ea506a2b 100644 --- a/internal/utils/tar.go +++ b/internal/utils/tar.go @@ -225,3 +225,58 @@ func TarAddDir(srcPath, prefix string, tarFile io.Writer) error { }) return err } + +// TarAddFile add a file to a tar file. The file is placed at dest path. +func TarAddFile(src, dest string, tarFile io.Writer) error { + tarWriter := tar.NewWriter(tarFile) + defer tarWriter.Close() + + fi, err := os.Stat(src) + if err != nil { + return err + } + + if fi.IsDir() { + return errors.New("source path is a directory") + } + + // generate tar header + header, err := tar.FileInfoHeader(fi, src) + if err != nil { + return err + } + + header.Name = dest + + // write header + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + data, err := os.Open(src) + if err != nil { + return err + } + if _, err := io.Copy(tarWriter, data); err != nil { + return err + } + return data.Close() +} + +func TarReadFile(src string, tarFile io.Reader) ([]byte, error) { + tarReader := tar.NewReader(tarFile) + + for { + header, err := tarReader.Next() + if err != nil { + return nil, err + } + if header.Name == src { + data, err := io.ReadAll(tarReader) + if err != nil { + return nil, err + } + return data, nil + } + } +} diff --git a/pkg/daemon/backup.go b/pkg/daemon/backup.go index 6aaf5afa..6d27e583 100644 --- a/pkg/daemon/backup.go +++ b/pkg/daemon/backup.go @@ -1,10 +1,6 @@ package daemon -import "github.com/NethermindEth/eigenlayer/internal/backup" - type BackupManager interface { // BackupInstance creates a backup of the instance with the given ID. BackupInstance(instanceId string) (string, error) - // BackupList returns a list of all backups. - BackupList() ([]backup.BackupInfo, error) } diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index c8ef3de9..4b957806 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -257,7 +257,11 @@ func (h HardwareRequirements) String() string { } type BackupInfo struct { + Id string Instance string Timestamp time.Time - SizeBytes uint64 + SizeBytes int64 + Version string + Commit string + Url string } diff --git a/pkg/daemon/egn_daemon.go b/pkg/daemon/egn_daemon.go index cf964fd3..ed3cee6d 100644 --- a/pkg/daemon/egn_daemon.go +++ b/pkg/daemon/egn_daemon.go @@ -1041,16 +1041,24 @@ func (d *EgnDaemon) Backup(instanceId string) (string, error) { } func (d *EgnDaemon) BackupList() ([]BackupInfo, error) { - backupInfo, err := d.backupManager.BackupList() + backups, err := d.dataDir.BackupList() if err != nil { return nil, err } - out := make([]BackupInfo, len(backupInfo)) - for i, b := range backupInfo { + out := make([]BackupInfo, len(backups)) + for i, b := range backups { + size, err := d.dataDir.BackupSize(b.Id()) + if err != nil { + return nil, err + } out[i] = BackupInfo{ - Instance: b.Instance, + Id: b.Id(), + Instance: b.InstanceId, Timestamp: b.Timestamp, - SizeBytes: b.SizeBytes, + SizeBytes: size, + Version: b.Version, + Commit: b.Commit, + Url: b.Url, } } return out, nil From cf546d0be866c24d6290c3fe607afd3bb1dd79cd Mon Sep 17 00:00:00 2001 From: adriantpaez Date: Wed, 4 Oct 2023 12:26:29 +0000 Subject: [PATCH 2/5] test: datadir backup list --- internal/data/datadir_test.go | 164 ++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/internal/data/datadir_test.go b/internal/data/datadir_test.go index 94a9f590..406baf09 100644 --- a/internal/data/datadir_test.go +++ b/internal/data/datadir_test.go @@ -1,10 +1,12 @@ package data import ( + "archive/tar" "errors" "fmt" "os" "path/filepath" + "strconv" "testing" "time" @@ -951,6 +953,139 @@ func TestMonitoringStack(t *testing.T) { verify(t, monitoringStack) } +func TestDataDir_BackupList(t *testing.T) { + type testData struct { + backup Backup + state []byte + timestamp time.Time + } + tc := []struct { + name string + data []testData + err error + }{ + { + name: "success", + data: []testData{ + { + backup: Backup{ + InstanceId: "mock-avs-default", + Timestamp: time.Unix(1696420902, 0), + Version: "v5.5.1", + Commit: "d5af645fffb93e8263b099082a4f512e1917d0af", + Url: "https://github.com/NethermindEth/mock-avs-pkg", + }, + state: []byte(` + { + "name": "mock-avs", + "url": "https://github.com/NethermindEth/mock-avs-pkg", + "version": "v5.5.1", + "spec_version": "v0.1.0", + "commit": "d5af645fffb93e8263b099082a4f512e1917d0af", + "profile": "option-returner", + "tag": "default", + "monitoring": { + "targets": [ + { + "service": "main-service", + "port": "8080", + "path": "/metrics" + } + ] + }, + "api": { + "service": "main-service", + "port": "8080" + }, + "plugin": { + "image": "mock-avs-plugin:v0.2.0" + } + } + `), + timestamp: time.Unix(1696420902, 0), + }, + { + backup: Backup{ + InstanceId: "mock-avs-second", + Timestamp: time.Unix(1696421646, 0), + Version: "v5.5.0", + Commit: "a3406616b848164358fdd24465b8eecda5f5ae34", + Url: "https://github.com/NethermindEth/mock-avs-pkg", + }, + state: []byte(` + { + "name": "mock-avs", + "url": "https://github.com/NethermindEth/mock-avs-pkg", + "version": "v5.5.0", + "spec_version": "v0.1.0", + "commit": "a3406616b848164358fdd24465b8eecda5f5ae34", + "profile": "option-returner", + "tag": "second", + "monitoring": { + "targets": [ + { + "service": "main-service", + "port": "8080", + "path": "/metrics" + } + ] + }, + "api": { + "service": "main-service", + "port": "8080" + }, + "plugin": { + "image": "mock-avs-plugin:v0.1.0" + } + } + `), + timestamp: time.Unix(1696421646, 0), + }, + }, + err: nil, + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewOsFs() + dataDirPath, err := afero.TempDir(fs, "", "datadir") + require.NoError(t, err) + dataDir, err := NewDataDir(dataDirPath, fs, nil) + require.NoError(t, err) + + backups := make([]Backup, 0, len(tt.data)) + + for _, d := range tt.data { + err = dataDir.InitBackup(&d.backup) + require.NoError(t, err) + backupTarPath := dataDir.BackupPath(d.backup.Id()) + backupTarFile, err := fs.OpenFile(backupTarPath, os.O_WRONLY, 0o644) + require.NoError(t, err) + tarWriter := tar.NewWriter(backupTarFile) + tarAddStateJson(t, tarWriter, d.state) + tarAddTimestamp(t, tarWriter, d.timestamp) + err = tarWriter.Close() + require.NoError(t, err) + backups = append(backups, d.backup) + } + + got, err := dataDir.BackupList() + + for i := 0; i < len(got); i++ { + got[i].Id() + } + + if tt.err != nil { + require.Error(t, err) + assert.EqualError(t, err, tt.err.Error()) + } else { + require.NoError(t, err) + assert.Equal(t, backups, got) + } + }) + } +} + func TestRemoveMonitoringStack(t *testing.T) { // Create monitoring stack // Create a memory filesystem @@ -995,3 +1130,32 @@ func TestRemoveMonitoringStackError(t *testing.T) { err = dataDir.RemoveMonitoringStack() require.ErrorIs(t, err, ErrMonitoringStackNotFound) } + +func tarAddStateJson(t *testing.T, tarWriter *tar.Writer, state []byte) { + t.Helper() + header := &tar.Header{ + Name: "data/state.json", + Size: int64(len(state)), + Mode: 0o644, + ModTime: time.Now(), + } + err := tarWriter.WriteHeader(header) + require.NoError(t, err) + _, err = tarWriter.Write(state) + require.NoError(t, err) +} + +func tarAddTimestamp(t *testing.T, tarWriter *tar.Writer, timestamp time.Time) { + t.Helper() + data := strconv.FormatInt(timestamp.Unix(), 10) + header := &tar.Header{ + Name: "timestamp", + Size: int64(len(data)), + Mode: 0o644, + ModTime: time.Now(), + } + err := tarWriter.WriteHeader(header) + require.NoError(t, err) + _, err = tarWriter.Write([]byte(data)) + require.NoError(t, err) +} From ec25e0fc702bf887a0c1da5554979ca78913b103 Mon Sep 17 00:00:00 2001 From: adriantpaez Date: Wed, 4 Oct 2023 12:45:38 +0000 Subject: [PATCH 3/5] test: add test cases for load timestamp and state --- internal/data/backup_test.go | 77 ++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/internal/data/backup_test.go b/internal/data/backup_test.go index 2d5c606d..92afff91 100644 --- a/internal/data/backup_test.go +++ b/internal/data/backup_test.go @@ -163,3 +163,80 @@ func TestBackupFromTar(t *testing.T) { }, *b) } + +func TestLoadBackupTarStateJson(t *testing.T) { + fs := afero.NewOsFs() + tarFile, err := afero.TempFile(fs, t.TempDir(), "backup-*.tar") + require.NoError(t, err) + defer tarFile.Close() + tarWriter := tar.NewWriter(tarFile) + tarAddStateJson(t, tarWriter, []byte(` + { + "name": "mock-avs", + "url": "https://github.com/NethermindEth/mock-avs-pkg", + "version": "v5.5.0", + "spec_version": "v0.1.0", + "commit": "a3406616b848164358fdd24465b8eecda5f5ae34", + "profile": "option-returner", + "tag": "second", + "monitoring": { + "targets": [ + { + "service": "main-service", + "port": "8080", + "path": "/metrics" + } + ] + }, + "api": { + "service": "main-service", + "port": "8080" + }, + "plugin": { + "image": "mock-avs-plugin:v0.1.0" + } + } + `)) + got, err := loadBackupTarStateJson(fs, tarFile.Name()) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, Instance{ + Name: "mock-avs", + Tag: "second", + URL: "https://github.com/NethermindEth/mock-avs-pkg", + Version: "v5.5.0", + SpecVersion: "v0.1.0", + Commit: "a3406616b848164358fdd24465b8eecda5f5ae34", + Profile: "option-returner", + MonitoringTargets: MonitoringTargets{ + Targets: []MonitoringTarget{ + { + Service: "main-service", + Port: "8080", + Path: "/metrics", + }, + }, + }, + APITarget: &APITarget{ + Service: "main-service", + Port: "8080", + }, + Plugin: &Plugin{ + Image: "mock-avs-plugin:v0.1.0", + }, + }, *got) +} + +func TestLoadBackupTarTimestamp(t *testing.T) { + fs := afero.NewOsFs() + tarFile, err := afero.TempFile(fs, t.TempDir(), "backup-*.tar") + require.NoError(t, err) + defer tarFile.Close() + tarWriter := tar.NewWriter(tarFile) + timestamp := time.Unix(1696367916, 0) + tarAddTimestamp(t, tarWriter, timestamp) + got, err := loadBackupTarTimestamp(fs, tarFile.Name()) + require.NoError(t, err) + require.NotNil(t, got) + assert.True(t, timestamp.Equal(got)) +} From 2a0e1993a5f1805998c757ac419994e8ff6234b9 Mon Sep 17 00:00:00 2001 From: adriantpaez Date: Wed, 4 Oct 2023 13:21:36 +0000 Subject: [PATCH 4/5] test: tar add file --- internal/utils/tar_test.go | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/internal/utils/tar_test.go b/internal/utils/tar_test.go index 0419b046..65568e7a 100644 --- a/internal/utils/tar_test.go +++ b/internal/utils/tar_test.go @@ -2,6 +2,7 @@ package utils import ( "archive/tar" + "bytes" "fmt" "io" "os" @@ -212,6 +213,76 @@ func TestTarAddDir(t *testing.T) { require.Equal(t, io.EOF, err, "expected EOF") } +func TestTarAddFile(t *testing.T) { + // Create temporary directory and file + tmpDir := t.TempDir() + tmpFile, err := os.CreateTemp(tmpDir, "test-*.txt") + require.NoError(t, err) + _, err = tmpFile.WriteString("This is a test file") + require.NoError(t, err) + + tests := []struct { + name string + src string + dest string + expected []byte + wantErr bool + }{ + { + name: "add file to tar", + src: tmpFile.Name(), + dest: "testfile.txt", + expected: []byte("This is a test file"), + wantErr: false, + }, + { + name: "add nonexistent file to tar", + src: filepath.Join(tmpDir, "nonexistent.txt"), + dest: "nonexistent.txt", + expected: nil, + wantErr: true, + }, + { + name: "add directory to tar", + src: tmpDir, + dest: "testdata", + expected: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temporary tar file + tmpTarFile, err := os.CreateTemp(t.TempDir(), "test-*.tar") + require.NoError(t, err) + defer os.Remove(tmpTarFile.Name()) + + // Add file to tar + err = TarAddFile(tt.src, tt.dest, tmpTarFile) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + // Read tar file + tmpTarFile.Seek(0, 0) + tr := tar.NewReader(tmpTarFile) + + // Check tar contents + header, err := tr.Next() + require.NoError(t, err) + assert.Equal(t, tt.dest, header.Name) + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, tr) + require.NoError(t, err) + assert.Equal(t, tt.expected, buf.Bytes()) + }) + } +} + func assertEqualDirs(t *testing.T, dir1, dir2 string) { err := filepath.Walk(dir1, func(path1 string, info1 os.FileInfo, err1 error) error { if err1 != nil { From 4dc03a18a52fb4f3e14af7dc9f021e6c4aab1d50 Mon Sep 17 00:00:00 2001 From: adriantpaez Date: Wed, 4 Oct 2023 13:48:45 +0000 Subject: [PATCH 5/5] doc: update README --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 65f31687..a00ff951 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ Eigenlayer is a setup wizard for EigenLayer Node Software. The tool installs, ma - [Running a Plugin](#running-a-plugin) - [Passing arguments to the plugin](#passing-arguments-to-the-plugin) - ## Dependencies This tool depends on [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) in order to manage the installation and running of EigenLayer nodes. Please make sure that you have both Docker and Docker Compose installed and configured properly before using this tool. @@ -43,7 +42,8 @@ First, install the Go programming language following the [official instructions] This command will install the `eigenlayer` executable along with the library and its dependencies in your system: > As the repository is private, you need to set the `GOPRIVATE` variable properly by running the following command: `export GOPRIVATE=github.com/NethermindEth/eigenlayer,$GOPRIVATE`. Git will automatically resolve the private access if your Git user has all the required permissions over the repository. -``` + +```bash go install github.com/NethermindEth/eigenlayer/cmd/eigenlayer@latest ``` @@ -71,7 +71,7 @@ go build -o build/eigenlayer cmd/eigenlayer/main.go or if you have `make` installed: -``` +```bash git clone https://github.com/NethermindEth/eigenlayer.git cd eigenlayer make build @@ -80,9 +80,9 @@ make build The executable will be in the `build` folder. --- -In case you want the binary in your PATH (or if you used the [Using Go](#using-go) method and you don't have `$GOBIN` in your PATH), please copy the binary to `/usr/local/bin`: +In case you want the binary in your PATH (or if you used the [Using Go](#install-eigenlayer-cli-using-go) method and you don't have `$GOBIN` in your PATH), please copy the binary to `/usr/local/bin`: -``` +```bash # Using Go sudo cp $GOPATH/bin/eigenlayer /usr/local/bin/ @@ -169,8 +169,8 @@ INFO[0002] The installed node software has a plugin. Notice the usage of: -* `--profile` to select the `option-returner` profile without prompt. -* `--no-prompt` to skip options prompts. +- `--profile` to select the `option-returner` profile without prompt. +- `--no-prompt` to skip options prompts. In this case, the `option-returner` profile uses all the default values. To set option values, use the `--option.` dynamic flags. For instance: @@ -230,7 +230,7 @@ eigenlayer update mock-avs-default Output: -``` +```log INFO[0000] Pulling package... INFO[0000] Package pulled successfully INFO[0000] Package version changed: v5.4.0 -> v5.5.0 @@ -275,7 +275,7 @@ eigenlayer backup mock-avs-default Output: -```bash +```log INFO[0000] Backing up instance mock-avs-default INFO[0000] Backing up instance data... INFO[0000] Backup created with id: mock-avs-default-1696337650 @@ -292,8 +292,8 @@ eigenlayer backup ls Output: ```bash -AVS Instance ID TIMESTAMP SIZE (GB) -mock-avs-default 2023-10-01 08:00:00 0.000009 +ID AVS Instance ID VERSION COMMIT TIMESTAMP SIZE URL +6ee67470 mock-avs-default v5.5.1 d5af645fffb93e8263b099082a4f512e1917d0af 2023-10-04 13:41:06 10KiB https://github.com/NethermindEth/mock-avs-pkg ``` ## Uninstalling AVS Node Software @@ -347,7 +347,7 @@ AVS instance logs could be retrieved using the `eigenlayer logs` command. Logs a eigenlayer logs mock-avs-default ``` -```bash +```log option-returner: INFO: Started server process [1] option-returner: INFO: Waiting for application startup. option-returner: INFO: Application startup complete. @@ -389,7 +389,7 @@ eigenlayer plugin mock-avs-default Output: -```bash +```log INFO[0001] Running plugin with image eigen-plugin-mock-avs-default on network eigenlayer INFO[0002] AVS is up @@ -415,4 +415,4 @@ INFO[0004] AVS is up ``` -In this case, the plugin container receives the `--port 8080` arguments. Note that this is not a flag of the `eigenlayer plugin` command. \ No newline at end of file +In this case, the plugin container receives the `--port 8080` arguments. Note that this is not a flag of the `eigenlayer plugin` command.