Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support state-sync #7225

Merged
merged 20 commits into from
Apr 19, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
970a53f
feat(cosmic-swingset): add after-commit action
mhofman Feb 12, 2023
37858cb
fix(cosmic-swingset): remove unused saved blockTime
mhofman Mar 8, 2023
be68431
feat(cosmic-swingset): export swingStore kvData to vstorage
mhofman Feb 10, 2023
5dcc44d
feat(cosmic-swingset): remove unnecessary explicit activityhash
mhofman Mar 11, 2023
726e188
feat(vstorage): Export sub tree
mhofman Feb 11, 2023
b1072d8
feat(cosmic-swingset): basic snapshot wiring
mhofman Feb 12, 2023
30340fb
Revert "fix: disable state-sync by default, until we've implemented it"
mhofman Feb 17, 2023
c5410dd
fix(cosmic-swingset): shutdown controller on exit
mhofman Feb 24, 2023
df304d5
feat(cosmic-swingset): add kernel-db exporter
mhofman Feb 16, 2023
22bc1d5
feat(cosmic-swingset): wire snapshot taking in chain-main
mhofman Feb 19, 2023
05e2b15
feat(cosmic-swingset): execute export in a subprocess
mhofman Mar 7, 2023
a2dabd1
feat(cosmic-swingset): explicit verbose option for export db
mhofman Mar 10, 2023
4cc25f5
feat(cosmic-proto): add state-sync artifacts proto
mhofman Feb 24, 2023
7b0d99e
feat(cosmos): wire swingset SnapshotExtension
mhofman Feb 24, 2023
00fab12
feat(cosmic-swingset): add kernel-db importer
mhofman Mar 8, 2023
c94e49d
feat(cosmos): wire swingset RestoreExtension
mhofman Mar 8, 2023
9d053b7
feat(cosmic-swingset): wire snapshot restoring in chain-main
mhofman Mar 8, 2023
2faa1fb
feat(deployment): Enable state-sync on validator nodes
mhofman Mar 14, 2023
27b97c1
feat(ci): Use state-sync in deployment loadgen test
mhofman Mar 14, 2023
283fded
feat(cosmos): Export KVData as artifact
mhofman Feb 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat(vstorage): Export sub tree
  • Loading branch information
mhofman committed Apr 19, 2023
commit 726e188481f4286583e48e6c3d87b654a7dd8d73
23 changes: 23 additions & 0 deletions golang/cosmos/x/vstorage/keeper/keeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,27 @@ func NewKeeper(storeKey sdk.StoreKey) Keeper {

// ExportStorage fetches all storage
func (k Keeper) ExportStorage(ctx sdk.Context) []*types.DataEntry {
return k.ExportStorageFromPrefix(ctx, "")
}

// ExportStorageFromPrefix fetches storage only under the supplied pathPrefix.
func (k Keeper) ExportStorageFromPrefix(ctx sdk.Context, pathPrefix string) []*types.DataEntry {
store := ctx.KVStore(k.storeKey)

if len(pathPrefix) > 0 {
if err := types.ValidatePath(pathPrefix); err != nil {
panic(err)
}
pathPrefix = pathPrefix + types.PathSeparator
}

// since vstorage encodes keys with a prefix indicating the number of path
// elements, exporting all entries under a given path cannot use a prefix
// iterator. Instead we iterate over the whole vstorage content and check
// whether each entry matches the path prefix. This choice assumes most
// entries will be exported. An alternative implementation would be to
// recursively list all children under the pathPrefix, and export them.

iterator := sdk.KVStorePrefixIterator(store, nil)
JimLarson marked this conversation as resolved.
Show resolved Hide resolved

exported := []*types.DataEntry{}
Expand All @@ -135,10 +154,14 @@ func (k Keeper) ExportStorage(ctx sdk.Context) []*types.DataEntry {
continue
}
path := types.EncodedKeyToPath(iterator.Key())
if !strings.HasPrefix(path, pathPrefix) {
continue
}
value, hasPrefix := cutPrefix(rawValue, types.EncodedDataPrefix)
if !hasPrefix {
panic(fmt.Errorf("value at path %q starts with unexpected prefix", path))
}
path = path[len(pathPrefix):]
entry := types.DataEntry{Path: path, Value: string(value)}
exported = append(exported, &entry)
}
Expand Down
18 changes: 14 additions & 4 deletions golang/cosmos/x/vstorage/keeper/keeper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,21 @@ func TestStorage(t *testing.T) {
{Path: "key2.child2.grandchild2", Value: "value2grandchild"},
{Path: "key2.child2.grandchild2a", Value: "value2grandchilda"},
}
got := keeper.ExportStorage(ctx)
if !reflect.DeepEqual(got, expectedExport) {
t.Errorf("got export %q, want %q", got, expectedExport)
gotExport := keeper.ExportStorage(ctx)
if !reflect.DeepEqual(gotExport, expectedExport) {
t.Errorf("got export %q, want %q", gotExport, expectedExport)
}
keeper.ImportStorage(ctx, got)

// Check the export.
expectedKey2Export := []*types.DataEntry{
{Path: "child2.grandchild2", Value: "value2grandchild"},
{Path: "child2.grandchild2a", Value: "value2grandchilda"},
}
if got := keeper.ExportStorageFromPrefix(ctx, "key2"); !reflect.DeepEqual(got, expectedKey2Export) {
t.Errorf("got export %q, want %q", got, expectedKey2Export)
}
Comment on lines +183 to +185
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if got := keeper.ExportStorageFromPrefix(ctx, "key2"); !reflect.DeepEqual(got, expectedKey2Export) {
t.Errorf("got export %q, want %q", got, expectedKey2Export)
}
if gotKey2Export := keeper.ExportStorageFromPrefix(ctx, "key2"); !reflect.DeepEqual(got, expectedKey2Export) {
t.Errorf("got prefix export %q, want %q", gotKey2Export, expectedKey2Export)
}


keeper.ImportStorage(ctx, gotExport)
}

func TestStorageNotify(t *testing.T) {
Expand Down