From 536e3cf213cdf97276459bd5659253d15ba88a9f Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Oct 2023 15:32:01 +0200 Subject: [PATCH] feat(migrations): Add option migration for inputs.disk --- config/migration.go | 31 ++++++- migrations/all/inputs_disk.go | 5 ++ migrations/inputs_disk/migration.go | 83 ++++++++++++++++++ migrations/inputs_disk/migration_test.go | 86 +++++++++++++++++++ .../deprecated_mountpoints/expected.conf | 3 + .../deprecated_mountpoints/telegraf.conf | 7 ++ .../expected.conf | 3 + .../telegraf.conf | 11 +++ migrations/registry.go | 14 +++ 9 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 migrations/all/inputs_disk.go create mode 100644 migrations/inputs_disk/migration.go create mode 100644 migrations/inputs_disk/migration_test.go create mode 100644 migrations/inputs_disk/testcases/deprecated_mountpoints/expected.conf create mode 100644 migrations/inputs_disk/testcases/deprecated_mountpoints/telegraf.conf create mode 100644 migrations/inputs_disk/testcases/deprecated_mountpoints_existing/expected.conf create mode 100644 migrations/inputs_disk/testcases/deprecated_mountpoints_existing/telegraf.conf diff --git a/config/migration.go b/config/migration.go index 453e7ade486c3..2e8029fa99bf9 100644 --- a/config/migration.go +++ b/config/migration.go @@ -151,7 +151,7 @@ func ApplyMigrations(data []byte) ([]byte, uint64, error) { return nil, 0, fmt.Errorf("assigning text failed: %w", err) } - // Do the actual migration(s) + // Do the actual plugin migration(s) var applied uint64 for idx, s := range sections { migrate, found := migrations.PluginMigrations[s.name] @@ -168,10 +168,39 @@ func ApplyMigrations(data []byte) ([]byte, uint64, error) { log.Printf("I! Plugin %q in line %d: %s", s.name, s.begin, msg) } s.raw = bytes.NewBuffer(result) + tbl, err := toml.Parse(s.raw.Bytes()) + if err != nil { + return nil, 0, fmt.Errorf("reparsing migrated %q (line %d) failed: %w", s.name, s.begin, err) + } + s.content = tbl + sections[idx] = s + applied++ + } + + // Do the actual plugin option migration(s) + for idx, s := range sections { + migrate, found := migrations.PluginOptionMigrations[s.name] + if !found { + continue + } + + log.Printf("D! migrating options of plugin %q in line %d...", s.name, s.begin) + result, msg, err := migrate(s.content) + if err != nil { + if errors.Is(err, migrations.ErrNotApplicable) { + continue + } + return nil, 0, fmt.Errorf("migrating options of %q (line %d) failed: %w", s.name, s.begin, err) + } + if msg != "" { + log.Printf("I! Plugin %q in line %d: %s", s.name, s.begin, msg) + } + s.raw = bytes.NewBuffer(result) sections[idx] = s applied++ } + // Reconstruct the config file from the sections var buf bytes.Buffer for _, s := range sections { _, err = s.raw.WriteTo(&buf) diff --git a/migrations/all/inputs_disk.go b/migrations/all/inputs_disk.go new file mode 100644 index 0000000000000..3742b75e01116 --- /dev/null +++ b/migrations/all/inputs_disk.go @@ -0,0 +1,5 @@ +//go:build !custom || (migrations && (inputs || inputs.disk)) + +package all + +import _ "github.com/influxdata/telegraf/migrations/inputs_disk" // register migration diff --git a/migrations/inputs_disk/migration.go b/migrations/inputs_disk/migration.go new file mode 100644 index 0000000000000..066083d10153b --- /dev/null +++ b/migrations/inputs_disk/migration.go @@ -0,0 +1,83 @@ +package inputs_disk + +import ( + "fmt" + + "github.com/influxdata/toml" + "github.com/influxdata/toml/ast" + + "github.com/influxdata/telegraf/internal/choice" + "github.com/influxdata/telegraf/migrations" +) + +// Migration function +func migrate(tbl *ast.Table) ([]byte, string, error) { + // Decode the old data structure + var plugin map[string]interface{} + if err := toml.UnmarshalTable(tbl, &plugin); err != nil { + return nil, "", err + } + + // Check for deprecated option(s) and migrate them + var applied bool + if rawDeprecatedMountpoints, found := plugin["mountpoints"]; found { + applied = true + + // Convert the options to the actual type + deprecatedMountpoints, ok := rawDeprecatedMountpoints.([]interface{}) + if !ok { + err := fmt.Errorf("unexpected type for deprecated 'mountpoints' option: %T", rawDeprecatedMountpoints) + return nil, "", err + } + + // Merge the option with the replacement + var mountpoints []string + if rawMountpoints, found := plugin["mount_points"]; found { + mountpointsList, ok := rawMountpoints.([]interface{}) + if !ok { + err := fmt.Errorf("unexpected type for 'mount_points' option: %T", rawMountpoints) + return nil, "", err + } + for _, raw := range mountpointsList { + mp, ok := raw.(string) + if !ok { + err := fmt.Errorf("unexpected type for 'mount_points' option: %T", raw) + return nil, "", err + } + mountpoints = append(mountpoints, mp) + } + } + for _, raw := range deprecatedMountpoints { + dmp, ok := raw.(string) + if !ok { + err := fmt.Errorf("unexpected type for deprecated 'mountpoints' option: %T", raw) + return nil, "", err + } + + if !choice.Contains(dmp, mountpoints) { + mountpoints = append(mountpoints, dmp) + } + } + + // Remove the deprecated option and replace the modified one + delete(plugin, "mountpoints") + plugin["mount_points"] = mountpoints + } + + // No options migrated so we can exit early + if !applied { + return nil, "", migrations.ErrNotApplicable + } + + // Create the corresponding plugin configurations + cfg := migrations.CreateTOMLStruct("inputs", "disk") + cfg.Add("inputs", "disk", plugin) + + output, err := toml.Marshal(cfg) + return output, "", err +} + +// Register the migration function for the plugin type +func init() { + migrations.AddPluginOptionMigration("inputs.disk", migrate) +} diff --git a/migrations/inputs_disk/migration_test.go b/migrations/inputs_disk/migration_test.go new file mode 100644 index 0000000000000..e460d8594ad88 --- /dev/null +++ b/migrations/inputs_disk/migration_test.go @@ -0,0 +1,86 @@ +package inputs_disk_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf/config" + _ "github.com/influxdata/telegraf/migrations/inputs_disk" // register migration + _ "github.com/influxdata/telegraf/plugins/inputs/disk" // register plugin +) + +func TestNoMigration(t *testing.T) { + defaultCfg := []byte(` +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## By default stats will be gathered for all mount points. + ## Set mount_points will restrict the stats to only the specified mount points. + # mount_points = ["/"] + + ## Ignore mount points by filesystem type. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + + ## Ignore mount points by mount options. + ## The 'mount' command reports options of all mounts in parathesis. + ## Bind mounts can be ignored with the special 'bind' option. + # ignore_mount_opts = [] +`) + + // Migrate and check that nothing changed + output, n, err := config.ApplyMigrations(defaultCfg) + require.NoError(t, err) + require.NotEmpty(t, output) + require.Zero(t, n) + require.Equal(t, string(defaultCfg), string(output)) +} + +func TestCases(t *testing.T) { + // Get all directories in testdata + folders, err := os.ReadDir("testcases") + require.NoError(t, err) + + for _, f := range folders { + // Only handle folders + if !f.IsDir() { + continue + } + + t.Run(f.Name(), func(t *testing.T) { + testcasePath := filepath.Join("testcases", f.Name()) + inputFile := filepath.Join(testcasePath, "telegraf.conf") + expectedFile := filepath.Join(testcasePath, "expected.conf") + + // Read the expected output + expected := config.NewConfig() + require.NoError(t, expected.LoadConfig(expectedFile)) + require.NotEmpty(t, expected.Inputs) + + // Read the input data + input, remote, err := config.LoadConfigFile(inputFile) + require.NoError(t, err) + require.False(t, remote) + require.NotEmpty(t, input) + + // Migrate + output, n, err := config.ApplyMigrations(input) + require.NoError(t, err) + require.NotEmpty(t, output) + require.GreaterOrEqual(t, n, uint64(1)) + actual := config.NewConfig() + require.NoError(t, actual.LoadConfigData(output)) + + // Test the output + require.Len(t, actual.Inputs, len(expected.Inputs)) + actualIDs := make([]string, 0, len(expected.Inputs)) + expectedIDs := make([]string, 0, len(expected.Inputs)) + for i := range actual.Inputs { + actualIDs = append(actualIDs, actual.Inputs[i].ID()) + expectedIDs = append(expectedIDs, expected.Inputs[i].ID()) + } + require.ElementsMatch(t, expectedIDs, actualIDs, string(output)) + }) + } +} diff --git a/migrations/inputs_disk/testcases/deprecated_mountpoints/expected.conf b/migrations/inputs_disk/testcases/deprecated_mountpoints/expected.conf new file mode 100644 index 0000000000000..fcc875cc6460a --- /dev/null +++ b/migrations/inputs_disk/testcases/deprecated_mountpoints/expected.conf @@ -0,0 +1,3 @@ +[[inputs.disk]] +mount_points = ["/mnt/disk", "/srv", "/mnt/others"] +ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] diff --git a/migrations/inputs_disk/testcases/deprecated_mountpoints/telegraf.conf b/migrations/inputs_disk/testcases/deprecated_mountpoints/telegraf.conf new file mode 100644 index 0000000000000..303294ef24c3d --- /dev/null +++ b/migrations/inputs_disk/testcases/deprecated_mountpoints/telegraf.conf @@ -0,0 +1,7 @@ +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## Deprecated mountpoint option + mountpoints = ["/mnt/disk", "/srv", "/mnt/others"] + + ## Ignore mount points by filesystem type. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] diff --git a/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/expected.conf b/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/expected.conf new file mode 100644 index 0000000000000..0c6e95656399b --- /dev/null +++ b/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/expected.conf @@ -0,0 +1,3 @@ +[[inputs.disk]] +mount_points = ["/", "/srv", "/mnt/disk", "/mnt/others"] +ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] diff --git a/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/telegraf.conf b/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/telegraf.conf new file mode 100644 index 0000000000000..54e906b03acc8 --- /dev/null +++ b/migrations/inputs_disk/testcases/deprecated_mountpoints_existing/telegraf.conf @@ -0,0 +1,11 @@ +# Read metrics about disk usage by mount point +[[inputs.disk]] + ## Deprecated mountpoint option + mountpoints = ["/mnt/disk", "/srv", "/mnt/others"] + + ## By default stats will be gathered for all mount points. + ## Set mount_points will restrict the stats to only the specified mount points. + mount_points = ["/", "/srv"] + + ## Ignore mount points by filesystem type. + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] diff --git a/migrations/registry.go b/migrations/registry.go index b03a663f75af1..abb838b5fc9f0 100644 --- a/migrations/registry.go +++ b/migrations/registry.go @@ -1,11 +1,14 @@ package migrations import ( + "errors" "fmt" "github.com/influxdata/toml/ast" ) +var ErrNotApplicable = errors.New("no migration applied") + type PluginMigrationFunc func(*ast.Table) ([]byte, string, error) var PluginMigrations = make(map[string]PluginMigrationFunc) @@ -17,6 +20,17 @@ func AddPluginMigration(name string, f PluginMigrationFunc) { PluginMigrations[name] = f } +type PluginOptionMigrationFunc PluginMigrationFunc + +var PluginOptionMigrations = make(map[string]PluginOptionMigrationFunc) + +func AddPluginOptionMigration(name string, f PluginOptionMigrationFunc) { + if _, found := PluginOptionMigrations[name]; found { + panic(fmt.Errorf("plugin option migration function already registered for %q", name)) + } + PluginOptionMigrations[name] = f +} + type pluginTOMLStruct map[string]map[string][]interface{} func CreateTOMLStruct(category, name string) pluginTOMLStruct {