Skip to content

Commit

Permalink
feat: compact the scheduling UI and use an enum for clock configurati…
Browse files Browse the repository at this point in the history 
…on (#452)
  • Loading branch information
@garethgeorge
garethgeorge authored Sep 4, 2024
1 parent b7b13cc commit 9205da1
Show file tree
Hide file tree
Showing 11 changed files with 448 additions and 550 deletions.
463 changes: 239 additions & 224 deletions gen/go/v1/config.pb.go
View file

Large diffs are not rendered by default.

23 changes: 2 additions & 21 deletions internal/config/migrations/003relativescheduling.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,6 @@ import (
v1 "github.com/garethgeorge/backrest/gen/go/v1"
)

func convertToRelativeSchedule(sched *v1.Schedule) {
switch s := sched.GetSchedule().(type) {
case *v1.Schedule_MaxFrequencyDays:
sched.Schedule = &v1.Schedule_MinDaysSinceLastRun{
MinDaysSinceLastRun: s.MaxFrequencyDays,
}
case *v1.Schedule_MaxFrequencyHours:
sched.Schedule = &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: s.MaxFrequencyHours,
}
case *v1.Schedule_Cron:
sched.Schedule = &v1.Schedule_CronSinceLastRun{
CronSinceLastRun: s.Cron,
}
default:
// do nothing
}
}

func migration003RelativeScheduling(config *v1.Config) {
// loop over plans and examine prune policy's
for _, repo := range config.Repos {
Expand All @@ -32,11 +13,11 @@ func migration003RelativeScheduling(config *v1.Config) {
}

if schedule := repo.GetPrunePolicy().GetSchedule(); schedule != nil {
convertToRelativeSchedule(schedule)
schedule.Clock = v1.Schedule_CLOCK_LAST_RUN_TIME
}

if schedule := repo.GetCheckPolicy().GetSchedule(); schedule != nil {
convertToRelativeSchedule(schedule)
schedule.Clock = v1.Schedule_CLOCK_LAST_RUN_TIME
}
}
}
94 changes: 4 additions & 90 deletions internal/config/migrations/003relativescheduling_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ func Test003Migration(t *testing.T) {
config := &v1.Config{
Repos: []*v1.Repo{
{
Id: "prune daily",
Id: "prune",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MaxFrequencyDays{
Expand All @@ -27,98 +27,12 @@ func Test003Migration(t *testing.T) {
},
},
},
{
Id: "prune hourly",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MaxFrequencyHours{
MaxFrequencyHours: 1,
},
},
},
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MaxFrequencyHours{
MaxFrequencyHours: 1,
},
},
},
},
{
Id: "prune cron",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_Cron{
Cron: "0 0 * * *",
},
},
},
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_Cron{
Cron: "0 0 * * *",
},
},
},
},
},
}

want := &v1.Config{
Repos: []*v1.Repo{
{
Id: "prune daily",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinDaysSinceLastRun{
MinDaysSinceLastRun: 1,
},
},
},
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinDaysSinceLastRun{
MinDaysSinceLastRun: 1,
},
},
},
},
{
Id: "prune hourly",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: 1,
},
},
},
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: 1,
},
},
},
},
{
Id: "prune cron",
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_CronSinceLastRun{
CronSinceLastRun: "0 0 * * *",
},
},
},
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_CronSinceLastRun{
CronSinceLastRun: "0 0 * * *",
},
},
},
},
},
}
want := proto.Clone(config).(*v1.Config)
want.Repos[0].PrunePolicy.Schedule.Clock = v1.Schedule_CLOCK_LAST_RUN_TIME
want.Repos[0].CheckPolicy.Schedule.Clock = v1.Schedule_CLOCK_LAST_RUN_TIME

migration003RelativeScheduling(config)

Expand Down
2 changes: 1 addition & 1 deletion internal/config/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func validatePlan(plan *v1.Plan, repos map[string]*v1.Repo) error {

if plan.Schedule != nil {
if e := protoutil.ValidateSchedule(plan.Schedule); e != nil {
err = multierror.Append(err, fmt.Errorf("schedule: %w", e))
err = multierror.Append(err, fmt.Errorf("backup schedule: %w", e))
}
}

Expand Down
5 changes: 2 additions & 3 deletions internal/orchestrator/orchestrator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, log *oplog.OpLog, logStor
cfg = proto.Clone(cfg).(*v1.Config)

// create the orchestrator.
var o *Orchestrator
o = &Orchestrator{
o := &Orchestrator{
OpLog: log,
config: cfg,
// repoPool created with a memory store to ensure the config is updated in an atomic operation with the repo pool's config value.
Expand Down Expand Up @@ -171,7 +170,7 @@ func (o *Orchestrator) ScheduleDefaultTasks(config *v1.Config) error {
}
}

zap.L().Info("reset task queue, scheduling new task set.")
zap.L().Info("reset task queue, scheduling new task set", zap.String("timezone", time.Now().Location().String()))

// Requeue tasks that are affected by the config change.
if err := o.ScheduleTask(tasks.NewCollectGarbageTask(), tasks.TaskPriorityDefault); err != nil {
Expand Down
52 changes: 42 additions & 10 deletions internal/orchestrator/tasks/scheduling_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,18 @@ func TestScheduling(t *testing.T) {
Id: "repo-relative",
CheckPolicy: &v1.CheckPolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: 1,
Schedule: &v1.Schedule_MaxFrequencyHours{
MaxFrequencyHours: 1,
},
Clock: v1.Schedule_CLOCK_LAST_RUN_TIME,
},
},
PrunePolicy: &v1.PrunePolicy{
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: 1,
Schedule: &v1.Schedule_MaxFrequencyHours{
MaxFrequencyHours: 1,
},
Clock: v1.Schedule_CLOCK_LAST_RUN_TIME,
},
},
},
Expand All @@ -64,14 +66,25 @@ func TestScheduling(t *testing.T) {
Schedule: &v1.Schedule_Cron{
Cron: "0 0 * * *", // every day at midnight
},
Clock: v1.Schedule_CLOCK_LOCAL,
},
},
{
Id: "plan-cron-utc",
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_Cron{
Cron: "0 0 * * *", // every day at midnight
},
Clock: v1.Schedule_CLOCK_UTC,
},
},
{
Id: "plan-cron-since-last-run",
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_CronSinceLastRun{
CronSinceLastRun: "0 0 * * *", // every day at midnight
Schedule: &v1.Schedule_Cron{
Cron: "0 0 * * *", // every day at midnight
},
Clock: v1.Schedule_CLOCK_LAST_RUN_TIME,
},
},
{
Expand All @@ -81,15 +94,17 @@ func TestScheduling(t *testing.T) {
Schedule: &v1.Schedule_MaxFrequencyDays{
MaxFrequencyDays: 1,
},
Clock: v1.Schedule_CLOCK_LOCAL,
},
},
{
Id: "plan-min-days-since-last-run",
Repo: "repo1",
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinDaysSinceLastRun{
MinDaysSinceLastRun: 1,
Schedule: &v1.Schedule_MaxFrequencyDays{
MaxFrequencyDays: 1,
},
Clock: v1.Schedule_CLOCK_LAST_RUN_TIME,
},
},
{
Expand All @@ -105,9 +120,10 @@ func TestScheduling(t *testing.T) {
Id: "plan-min-hours-since-last-run",
Repo: "repo1",
Schedule: &v1.Schedule{
Schedule: &v1.Schedule_MinHoursSinceLastRun{
MinHoursSinceLastRun: 1,
Schedule: &v1.Schedule_MaxFrequencyHours{
MaxFrequencyHours: 1,
},
Clock: v1.Schedule_CLOCK_LAST_RUN_TIME,
},
},
},
Expand Down Expand Up @@ -202,6 +218,22 @@ func TestScheduling(t *testing.T) {
},
wantTime: mustParseTime(t, "1970-01-02T00:00:00-08:00"),
},
{
name: "backup schedule cron utc",
task: NewScheduledBackupTask(config.FindPlan(cfg, "plan-cron-utc")),
ops: []*v1.Operation{
{
InstanceId: "instance1",
RepoId: "repo1",
PlanId: "plan-cron-utc",
Op: &v1.Operation_OperationBackup{
OperationBackup: &v1.OperationBackup{},
},
UnixTimeEndMs: farFuture.UnixMilli(),
},
},
wantTime: mustParseTime(t, "1970-01-02T08:00:00Z"),
},
{
name: "backup schedule cron since last run",
task: NewScheduledBackupTask(config.FindPlan(cfg, "plan-cron-since-last-run")),
Expand Down
48 changes: 18 additions & 30 deletions internal/protoutil/schedule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,31 @@ var ErrScheduleDisabled = errors.New("never")
// ResolveSchedule resolves a schedule to the next time it should run based on last execution.
// note that this is different from backup behavior which is always relative to the current time.
func ResolveSchedule(sched *v1.Schedule, lastRan time.Time, curTime time.Time) (time.Time, error) {
var t time.Time
switch sched.GetClock() {
case v1.Schedule_CLOCK_DEFAULT, v1.Schedule_CLOCK_LOCAL:
t = curTime.Local()
case v1.Schedule_CLOCK_UTC:
t = curTime.UTC()
case v1.Schedule_CLOCK_LAST_RUN_TIME:
t = lastRan
default:
return time.Time{}, fmt.Errorf("unknown clock type: %v", sched.GetClock().String())
}

switch s := sched.GetSchedule().(type) {
case *v1.Schedule_Disabled:
case *v1.Schedule_Disabled, nil:
return time.Time{}, ErrScheduleDisabled
case *v1.Schedule_MaxFrequencyDays:
return curTime.Add(time.Duration(s.MaxFrequencyDays) * 24 * time.Hour), nil
return t.Add(time.Duration(s.MaxFrequencyDays) * 24 * time.Hour), nil
case *v1.Schedule_MaxFrequencyHours:
return curTime.Add(time.Duration(s.MaxFrequencyHours) * time.Hour), nil
return t.Add(time.Duration(s.MaxFrequencyHours) * time.Hour), nil
case *v1.Schedule_Cron:
cron, err := cronexpr.ParseInLocation(s.Cron, time.Now().Location().String())
if err != nil {
return time.Time{}, fmt.Errorf("parse cron %q: %w", s.Cron, err)
}
return cron.Next(curTime), nil
case *v1.Schedule_MinHoursSinceLastRun:
return lastRan.Add(time.Duration(s.MinHoursSinceLastRun) * time.Hour), nil
case *v1.Schedule_MinDaysSinceLastRun:
return lastRan.Add(time.Duration(s.MinDaysSinceLastRun) * 24 * time.Hour), nil
case *v1.Schedule_CronSinceLastRun:
cron, err := cronexpr.ParseInLocation(s.CronSinceLastRun, time.Now().Location().String())
if err != nil {
return time.Time{}, fmt.Errorf("parse cron %q: %w", s.CronSinceLastRun, err)
}
return cron.Next(lastRan), nil
return cron.Next(t), nil
default:
return time.Time{}, fmt.Errorf("unknown schedule type: %T", s)
}
Expand All @@ -60,22 +62,8 @@ func ValidateSchedule(sched *v1.Schedule) error {
if err != nil {
return fmt.Errorf("invalid cron %q: %w", s.Cron, err)
}
case *v1.Schedule_MinHoursSinceLastRun:
if s.MinHoursSinceLastRun < 1 {
return errors.New("invalid min hours since last run")
}
case *v1.Schedule_MinDaysSinceLastRun:
if s.MinDaysSinceLastRun < 1 {
return errors.New("invalid min days since last run")
}
case *v1.Schedule_CronSinceLastRun:
if s.CronSinceLastRun == "" {
return errors.New("empty cron expression")
}
_, err := cronexpr.ParseInLocation(s.CronSinceLastRun, time.Now().Location().String())
if err != nil {
return fmt.Errorf("invalid cron %q: %w", s.CronSinceLastRun, err)
}
case nil:
return nil
case *v1.Schedule_Disabled:
if !s.Disabled {
return errors.New("disabled boolean must be set to true")
Expand Down
12 changes: 9 additions & 3 deletions proto/v1/config.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,16 @@ message Schedule {
string cron = 2 [json_name="cron"]; // cron expression describing the schedule.
int32 maxFrequencyDays = 3 [json_name="maxFrequencyDays"]; // max frequency of runs in days.
int32 maxFrequencyHours = 4 [json_name="maxFrequencyHours"]; // max frequency of runs in hours.
string cronSinceLastRun = 100 [json_name="cronSinceLastRun"]; // cron expression to run since the last run.
int32 minHoursSinceLastRun = 101 [json_name="minHoursSinceLastRun"]; // max hours since the last run.
int32 minDaysSinceLastRun = 102 [json_name="minDaysSinceLastRun"]; // max days since the last run.
}

enum Clock {
CLOCK_DEFAULT = 0; // same as CLOCK_LOCAL
CLOCK_LOCAL = 1;
CLOCK_UTC = 2;
CLOCK_LAST_RUN_TIME = 3;
}

Clock clock = 5 [json_name="clock"]; // clock to use for scheduling.
}

message Hook {
Expand Down
Loading

0 comments on commit 9205da1

Please sign in to comment.