-
-
Notifications
You must be signed in to change notification settings - Fork 554
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migrate serializer:gob fields to serializer:json (#1855)
- Loading branch information
Showing
6 changed files
with
290 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package migration | ||
|
|
||
| import ( | ||
| "encoding/gob" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| log "github.com/sirupsen/logrus" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| // MigrateFieldFromGobToJSON migrates a column from Gob encoding to JSON encoding. | ||
| // T is the type of the model that contains the field to be migrated. | ||
| // S is the type of the field to be migrated. | ||
| func MigrateFieldFromGobToJSON[T any, S any](db *gorm.DB, fieldName string) error { | ||
|
|
||
| oldColumnName := fieldName | ||
| newColumnName := fieldName + "_tmp" | ||
|
|
||
| var model T | ||
|
|
||
| if !db.Migrator().HasTable(&model) { | ||
| log.Debugf("Table for %T does not exist, no migration needed", model) | ||
| return nil | ||
| } | ||
|
|
||
| stmt := &gorm.Statement{DB: db} | ||
| err := stmt.Parse(model) | ||
| if err != nil { | ||
| return fmt.Errorf("parse model: %w", err) | ||
| } | ||
| tableName := stmt.Schema.Table | ||
|
|
||
| var item string | ||
| if err := db.Model(model).Select(oldColumnName).First(&item).Error; err != nil { | ||
| if errors.Is(err, gorm.ErrRecordNotFound) { | ||
| log.Debugf("No records in table %s, no migration needed", tableName) | ||
| return nil | ||
| } | ||
| return fmt.Errorf("fetch first record: %w", err) | ||
| } | ||
|
|
||
| var js json.RawMessage | ||
| var syntaxError *json.SyntaxError | ||
| err = json.Unmarshal([]byte(item), &js) | ||
| if err == nil || !errors.As(err, &syntaxError) { | ||
| log.Debugf("No migration needed for %s, %s", tableName, fieldName) | ||
| return nil | ||
| } | ||
|
|
||
| if err := db.Transaction(func(tx *gorm.DB) error { | ||
| if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", tableName, newColumnName)).Error; err != nil { | ||
| return fmt.Errorf("add column %s: %w", newColumnName, err) | ||
| } | ||
|
|
||
| var rows []map[string]any | ||
| if err := tx.Table(tableName).Select("id", oldColumnName).Find(&rows).Error; err != nil { | ||
| return fmt.Errorf("find rows: %w", err) | ||
| } | ||
|
|
||
| for _, row := range rows { | ||
| var field S | ||
|
|
||
| str, ok := row[oldColumnName].(string) | ||
| if !ok { | ||
| return fmt.Errorf("type assertion failed") | ||
| } | ||
| reader := strings.NewReader(str) | ||
|
|
||
| if err := gob.NewDecoder(reader).Decode(&field); err != nil { | ||
| return fmt.Errorf("gob decode error: %w", err) | ||
| } | ||
|
|
||
| jsonValue, err := json.Marshal(field) | ||
| if err != nil { | ||
| return fmt.Errorf("re-encode to JSON: %w", err) | ||
| } | ||
|
|
||
| if err := tx.Table(tableName).Where("id = ?", row["id"]).Update(newColumnName, jsonValue).Error; err != nil { | ||
| return fmt.Errorf("update row: %w", err) | ||
| } | ||
| } | ||
|
|
||
| if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, oldColumnName)).Error; err != nil { | ||
| return fmt.Errorf("drop column %s: %w", oldColumnName, err) | ||
| } | ||
| if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME COLUMN %s TO %s", tableName, newColumnName, oldColumnName)).Error; err != nil { | ||
| return fmt.Errorf("rename column %s to %s: %w", newColumnName, oldColumnName, err) | ||
| } | ||
|
|
||
| return nil | ||
| }); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| log.Infof("Migration of %s.%s from gob to json completed", tableName, fieldName) | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package migration_test | ||
|
|
||
| import ( | ||
| "encoding/gob" | ||
| "net" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| "gorm.io/driver/sqlite" | ||
| "gorm.io/gorm" | ||
|
|
||
| "github.com/netbirdio/netbird/management/server" | ||
| "github.com/netbirdio/netbird/management/server/migration" | ||
| "github.com/netbirdio/netbird/route" | ||
| ) | ||
|
|
||
| func setupDatabase(t *testing.T) *gorm.DB { | ||
| t.Helper() | ||
|
|
||
| db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{ | ||
| PrepareStmt: true, | ||
| }) | ||
|
|
||
| require.NoError(t, err, "Failed to open database") | ||
| return db | ||
| } | ||
|
|
||
| func TestMigrateFieldFromGobToJSON_EmptyDB(t *testing.T) { | ||
| db := setupDatabase(t) | ||
| err := migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net") | ||
| require.NoError(t, err, "Migration should not fail for an empty database") | ||
| } | ||
|
|
||
| func TestMigrateFieldFromGobToJSON_WithGobData(t *testing.T) { | ||
| db := setupDatabase(t) | ||
|
|
||
| err := db.AutoMigrate(&server.Account{}, &route.Route{}) | ||
| require.NoError(t, err, "Failed to auto-migrate tables") | ||
|
|
||
| _, ipnet, err := net.ParseCIDR("10.0.0.0/24") | ||
| require.NoError(t, err, "Failed to parse CIDR") | ||
|
|
||
| type network struct { | ||
| server.Network | ||
| Net net.IPNet `gorm:"serializer:gob"` | ||
| } | ||
|
|
||
| type account struct { | ||
| server.Account | ||
| Network *network `gorm:"embedded;embeddedPrefix:network_"` | ||
| } | ||
|
|
||
| err = db.Save(&account{Account: server.Account{Id: "123"}, Network: &network{Net: *ipnet}}).Error | ||
| require.NoError(t, err, "Failed to insert Gob data") | ||
|
|
||
| var gobStr string | ||
| err = db.Model(&server.Account{}).Select("network_net").First(&gobStr).Error | ||
| assert.NoError(t, err, "Failed to fetch Gob data") | ||
|
|
||
| err = gob.NewDecoder(strings.NewReader(gobStr)).Decode(&ipnet) | ||
| require.NoError(t, err, "Failed to decode Gob data") | ||
|
|
||
| err = migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net") | ||
| require.NoError(t, err, "Migration should not fail with Gob data") | ||
|
|
||
| var jsonStr string | ||
| db.Model(&server.Account{}).Select("network_net").First(&jsonStr) | ||
| assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be migrated") | ||
| } | ||
|
|
||
| func TestMigrateFieldFromGobToJSON_WithJSONData(t *testing.T) { | ||
| db := setupDatabase(t) | ||
|
|
||
| err := db.AutoMigrate(&server.Account{}, &route.Route{}) | ||
| require.NoError(t, err, "Failed to auto-migrate tables") | ||
|
|
||
| _, ipnet, err := net.ParseCIDR("10.0.0.0/24") | ||
| require.NoError(t, err, "Failed to parse CIDR") | ||
|
|
||
| err = db.Save(&server.Account{Network: &server.Network{Net: *ipnet}}).Error | ||
| require.NoError(t, err, "Failed to insert JSON data") | ||
|
|
||
| err = migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net") | ||
| require.NoError(t, err, "Migration should not fail with JSON data") | ||
|
|
||
| var jsonStr string | ||
| db.Model(&server.Account{}).Select("network_net").First(&jsonStr) | ||
| assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be unchanged") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters