diff --git a/go.mod b/go.mod index 688f59fe55..a7935a9c3c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/kr/pretty v0.3.1 github.com/leanovate/gopter v0.2.9 github.com/logrusorgru/aurora/v4 v4.0.0 - github.com/onflow/atree v0.6.1-0.20230706233410-78f997992600 + github.com/onflow/atree v0.6.1-0.20231024210403-cb829958cc5f github.com/rivo/uniseg v0.4.4 github.com/schollz/progressbar/v3 v3.13.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 6589d6c473..3da58a8aa0 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/mattn/go-tty v0.0.4/go.mod h1:u5GGXBtZU6RQoKV8gY5W6UhMudbR5vXnUe7j3px github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/onflow/atree v0.6.1-0.20230706233410-78f997992600 h1:OOnC8buJso6dSZ6W+RGQDnJW9UTh7HfEGyS/ivXJ7NI= -github.com/onflow/atree v0.6.1-0.20230706233410-78f997992600/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= +github.com/onflow/atree v0.6.1-0.20231024210403-cb829958cc5f h1:mmc7WjCGwiEkt1GlANRRoxxJRz469gBG0ik755gJJpg= +github.com/onflow/atree v0.6.1-0.20231024210403-cb829958cc5f/go.mod h1:7YNAyCd5JENq+NzH+fR1ABUZVzbSq9dkt0+5fZH3L2A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/term v1.2.0-beta.2 h1:L3y/h2jkuBVFdWiJvNfYfKmzcCnILw7mJWm2JQuMppw= github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= diff --git a/runtime/cmd/decode-state-values/main.go b/runtime/cmd/decode-state-values/main.go index 54867311c3..46830fc999 100644 --- a/runtime/cmd/decode-state-values/main.go +++ b/runtime/cmd/decode-state-values/main.go @@ -39,6 +39,7 @@ import ( "github.com/schollz/progressbar/v3" "github.com/onflow/cadence/runtime/common" + runtimeErr "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" ) @@ -91,8 +92,12 @@ func storageKeyToSlabID(address atree.Address, key string) atree.SlabID { return atree.NewSlabID(address, index) } -func decodeStorable(decoder *cbor.StreamDecoder, storableSlabStorageID atree.SlabID) (atree.Storable, error) { - return interpreter.DecodeStorable(decoder, storableSlabStorageID, nil) +func decodeStorable( + decoder *cbor.StreamDecoder, + storableSlabStorageID atree.SlabID, + inlinedExtraData []atree.ExtraData, +) (atree.Storable, error) { + return interpreter.DecodeStorable(decoder, storableSlabStorageID, inlinedExtraData, nil) } func decodeTypeInfo(decoder *cbor.StreamDecoder) (atree.TypeInfo, error) { @@ -136,6 +141,11 @@ type slabStorage struct{} var _ atree.SlabStorage = &slabStorage{} +func (s *slabStorage) RetrieveIfLoaded(atree.SlabID) atree.Slab { + // RetrieveIfLoaded() is used for loaded resource tracking. So it isn't needed here. + panic(runtimeErr.NewUnreachableError()) +} + func (s *slabStorage) Retrieve(id atree.SlabID) (atree.Slab, bool, error) { data, ok := storage[slabIDToStorageKey(id)] if !ok { @@ -350,7 +360,7 @@ func loadStorageKey( reader := bytes.NewReader(data) decoder := interpreter.CBORDecMode.NewStreamDecoder(reader) - storable, err := interpreter.DecodeStorable(decoder, atree.SlabIDUndefined, nil) + storable, err := interpreter.DecodeStorable(decoder, atree.SlabIDUndefined, nil, nil) if err != nil { log.Printf( "Failed to decode storable @ 0x%x %s: %s (data: %x)\n", diff --git a/runtime/convertValues.go b/runtime/convertValues.go index 82a2caa0c7..adddeef123 100644 --- a/runtime/convertValues.go +++ b/runtime/convertValues.go @@ -562,40 +562,44 @@ func exportDictionaryValue( var err error pairs := make([]cadence.KeyValuePair, 0, v.Count()) - v.Iterate(inter, func(key, value interpreter.Value) (resume bool) { - - var convertedKey cadence.Value - convertedKey, err = exportValueWithInterpreter( - key, - inter, - locationRange, - seenReferences, - ) - if err != nil { - return false - } + v.Iterate( + inter, + locationRange, + func(key, value interpreter.Value) (resume bool) { + + var convertedKey cadence.Value + convertedKey, err = exportValueWithInterpreter( + key, + inter, + locationRange, + seenReferences, + ) + if err != nil { + return false + } - var convertedValue cadence.Value - convertedValue, err = exportValueWithInterpreter( - value, - inter, - locationRange, - seenReferences, - ) - if err != nil { - return false - } + var convertedValue cadence.Value + convertedValue, err = exportValueWithInterpreter( + value, + inter, + locationRange, + seenReferences, + ) + if err != nil { + return false + } - pairs = append( - pairs, - cadence.KeyValuePair{ - Key: convertedKey, - Value: convertedValue, - }, - ) + pairs = append( + pairs, + cadence.KeyValuePair{ + Key: convertedKey, + Value: convertedValue, + }, + ) - return true - }) + return true + }, + ) if err != nil { return nil, err diff --git a/runtime/interpreter/decode.go b/runtime/interpreter/decode.go index 853494fd5b..97282f054d 100644 --- a/runtime/interpreter/decode.go +++ b/runtime/interpreter/decode.go @@ -118,23 +118,39 @@ func decodeInt64(d StorableDecoder) (int64, error) { func DecodeStorable( decoder *cbor.StreamDecoder, slabID atree.SlabID, + inlinedExtraData []atree.ExtraData, memoryGauge common.MemoryGauge, ) ( atree.Storable, error, ) { - return NewStorableDecoder(decoder, slabID, memoryGauge).decodeStorable() + return NewStorableDecoder(decoder, slabID, inlinedExtraData, memoryGauge).decodeStorable() +} + +func newStorableDecoderFunc(memoryGauge common.MemoryGauge) atree.StorableDecoder { + return func( + decoder *cbor.StreamDecoder, + slabID atree.SlabID, + inlinedExtraData []atree.ExtraData, + ) ( + atree.Storable, + error, + ) { + return NewStorableDecoder(decoder, slabID, inlinedExtraData, memoryGauge).decodeStorable() + } } func NewStorableDecoder( decoder *cbor.StreamDecoder, slabID atree.SlabID, + inlinedExtraData []atree.ExtraData, memoryGauge common.MemoryGauge, ) StorableDecoder { return StorableDecoder{ - decoder: decoder, - memoryGauge: memoryGauge, - slabID: slabID, + decoder: decoder, + memoryGauge: memoryGauge, + slabID: slabID, + inlinedExtraData: inlinedExtraData, TypeDecoder: NewTypeDecoder( decoder, memoryGauge, @@ -144,9 +160,10 @@ func NewStorableDecoder( type StorableDecoder struct { TypeDecoder - memoryGauge common.MemoryGauge - decoder *cbor.StreamDecoder - slabID atree.SlabID + memoryGauge common.MemoryGauge + decoder *cbor.StreamDecoder + slabID atree.SlabID + inlinedExtraData []atree.ExtraData } func (d StorableDecoder) decodeStorable() (atree.Storable, error) { @@ -203,6 +220,29 @@ func (d StorableDecoder) decodeStorable() (atree.Storable, error) { case atree.CBORTagSlabID: return atree.DecodeSlabIDStorable(d.decoder) + case atree.CBORTagInlinedArray: + return atree.DecodeInlinedArrayStorable( + d.decoder, + newStorableDecoderFunc(d.memoryGauge), + d.slabID, + d.inlinedExtraData) + + case atree.CBORTagInlinedMap: + return atree.DecodeInlinedMapStorable( + d.decoder, + newStorableDecoderFunc(d.memoryGauge), + d.slabID, + d.inlinedExtraData, + ) + + case atree.CBORTagInlinedCompactMap: + return atree.DecodeInlinedCompactMapStorable( + d.decoder, + newStorableDecoderFunc(d.memoryGauge), + d.slabID, + d.inlinedExtraData, + ) + case CBORTagVoidValue: err := d.decoder.Skip() if err != nil { @@ -225,6 +265,9 @@ func (d StorableDecoder) decodeStorable() (atree.Storable, error) { case CBORTagSomeValue: storable, err = d.decodeSome() + case CBORTagSomeValueWithNestedLevels: + storable, err = d.decodeSomeWithNestedLevels() + case CBORTagAddressValue: storable, err = d.decodeAddress() @@ -825,6 +868,59 @@ func (d StorableDecoder) decodeSome() (SomeStorable, error) { }, nil } +func (d StorableDecoder) decodeSomeWithNestedLevels() (SomeStorable, error) { + count, err := d.decoder.DecodeArrayHead() + if err != nil { + return SomeStorable{}, errors.NewUnexpectedError( + "invalid some value with nested levels encoding: %w", + err, + ) + } + + if count != someStorableWithMultipleNestedLevelsArrayCount { + return SomeStorable{}, errors.NewUnexpectedError( + "invalid array count for some value with nested levels encoding: got %d, expect %d", + count, someStorableWithMultipleNestedLevelsArrayCount, + ) + } + + nestedLevels, err := d.decoder.DecodeUint64() + if err != nil { + return SomeStorable{}, errors.NewUnexpectedError( + "invalid nested levels for some value with nested levels encoding: %w", + err, + ) + } + + if nestedLevels <= 1 { + return SomeStorable{}, errors.NewUnexpectedError( + "invalid nested levels for some value with nested levels encoding: got %d, expect > 1", + nestedLevels, + ) + } + + nonSomeStorable, err := d.decodeStorable() + if err != nil { + return SomeStorable{}, errors.NewUnexpectedError( + "invalid nonSomeStorable for some value with nested levels encoding: %w", + err, + ) + } + + storable := SomeStorable{ + gauge: d.memoryGauge, + Storable: nonSomeStorable, + } + for i := uint64(1); i < nestedLevels; i++ { + storable = SomeStorable{ + gauge: d.memoryGauge, + Storable: storable, + } + } + + return storable, nil +} + func checkEncodedAddressLength(actualLength int) error { const expectedLength = common.AddressLength if actualLength > expectedLength { @@ -1560,10 +1656,10 @@ func (d TypeDecoder) decodeInterfaceStaticType() (InterfaceStaticType, error) { return NewInterfaceStaticTypeComputeTypeID(d.memoryGauge, location, qualifiedIdentifier), nil } -func (d TypeDecoder) decodeVariableSizedStaticType() (StaticType, error) { +func (d TypeDecoder) decodeVariableSizedStaticType() (VariableSizedStaticType, error) { staticType, err := d.DecodeStaticType() if err != nil { - return nil, errors.NewUnexpectedError( + return VariableSizedStaticType{}, errors.NewUnexpectedError( "invalid variable-sized static type encoding: %w", err, ) @@ -1571,24 +1667,24 @@ func (d TypeDecoder) decodeVariableSizedStaticType() (StaticType, error) { return NewVariableSizedStaticType(d.memoryGauge, staticType), nil } -func (d TypeDecoder) decodeConstantSizedStaticType() (StaticType, error) { +func (d TypeDecoder) decodeConstantSizedStaticType() (ConstantSizedStaticType, error) { const expectedLength = encodedConstantSizedStaticTypeLength arraySize, err := d.decoder.DecodeArrayHead() if err != nil { if e, ok := err.(*cbor.WrongTypeError); ok { - return nil, errors.NewUnexpectedError( + return ConstantSizedStaticType{}, errors.NewUnexpectedError( "invalid constant-sized static type encoding: expected [%d]any, got %s", expectedLength, e.ActualType.String(), ) } - return nil, err + return ConstantSizedStaticType{}, err } if arraySize != expectedLength { - return nil, errors.NewUnexpectedError( + return ConstantSizedStaticType{}, errors.NewUnexpectedError( "invalid constant-sized static type encoding: expected [%d]any, got [%d]any", expectedLength, arraySize, @@ -1599,17 +1695,17 @@ func (d TypeDecoder) decodeConstantSizedStaticType() (StaticType, error) { size, err := decodeUint64(d.decoder, d.memoryGauge) if err != nil { if e, ok := err.(*cbor.WrongTypeError); ok { - return nil, errors.NewUnexpectedError( + return ConstantSizedStaticType{}, errors.NewUnexpectedError( "invalid constant-sized static type size encoding: %s", e.ActualType.String(), ) } - return nil, err + return ConstantSizedStaticType{}, err } const max = math.MaxInt64 if size > max { - return nil, errors.NewUnexpectedError( + return ConstantSizedStaticType{}, errors.NewUnexpectedError( "invalid constant-sized static type size: got %d, expected max %d", size, max, @@ -1619,7 +1715,7 @@ func (d TypeDecoder) decodeConstantSizedStaticType() (StaticType, error) { // Decode type at array index encodedConstantSizedStaticTypeTypeFieldKey staticType, err := d.DecodeStaticType() if err != nil { - return nil, errors.NewUnexpectedError( + return ConstantSizedStaticType{}, errors.NewUnexpectedError( "invalid constant-sized static type inner type encoding: %w", err, ) @@ -1685,24 +1781,24 @@ func (d TypeDecoder) decodeReferenceStaticType() (StaticType, error) { ), nil } -func (d TypeDecoder) decodeDictionaryStaticType() (StaticType, error) { +func (d TypeDecoder) decodeDictionaryStaticType() (DictionaryStaticType, error) { const expectedLength = encodedDictionaryStaticTypeLength arraySize, err := d.decoder.DecodeArrayHead() if err != nil { if e, ok := err.(*cbor.WrongTypeError); ok { - return nil, errors.NewUnexpectedError( + return DictionaryStaticType{}, errors.NewUnexpectedError( "invalid dictionary static type encoding: expected [%d]any, got %s", expectedLength, e.ActualType.String(), ) } - return nil, err + return DictionaryStaticType{}, err } if arraySize != expectedLength { - return nil, errors.NewUnexpectedError( + return DictionaryStaticType{}, errors.NewUnexpectedError( "invalid dictionary static type encoding: expected [%d]any, got [%d]any", expectedLength, arraySize, @@ -1712,7 +1808,7 @@ func (d TypeDecoder) decodeDictionaryStaticType() (StaticType, error) { // Decode key type at array index encodedDictionaryStaticTypeKeyTypeFieldKey keyType, err := d.DecodeStaticType() if err != nil { - return nil, errors.NewUnexpectedError( + return DictionaryStaticType{}, errors.NewUnexpectedError( "invalid dictionary static type key type encoding: %w", err, ) @@ -1721,7 +1817,7 @@ func (d TypeDecoder) decodeDictionaryStaticType() (StaticType, error) { // Decode value type at array index encodedDictionaryStaticTypeValueTypeFieldKey valueType, err := d.DecodeStaticType() if err != nil { - return nil, errors.NewUnexpectedError( + return DictionaryStaticType{}, errors.NewUnexpectedError( "invalid dictionary static type value type encoding: %w", err, ) diff --git a/runtime/interpreter/deepcopyremove_test.go b/runtime/interpreter/deepcopyremove_test.go index b1f2e6f6b4..9c6339e390 100644 --- a/runtime/interpreter/deepcopyremove_test.go +++ b/runtime/interpreter/deepcopyremove_test.go @@ -85,7 +85,7 @@ func TestValueDeepCopyAndDeepRemove(t *testing.T) { optionalValue, ) - compositeValue.DeepRemove(inter) + compositeValue.DeepRemove(inter, true) // Only count non-temporary slabs, // i.e. ones which have a non-empty address diff --git a/runtime/interpreter/encode.go b/runtime/interpreter/encode.go index 26565e4319..81733aa1ed 100644 --- a/runtime/interpreter/encode.go +++ b/runtime/interpreter/encode.go @@ -116,7 +116,7 @@ const ( _ // DO *NOT* REPLACE. Previously used for array values CBORTagStringValue CBORTagCharacterValue - _ + CBORTagSomeValueWithNestedLevels _ _ _ @@ -681,13 +681,23 @@ func (v UFix64Value) Encode(e *atree.Encoder) error { return e.CBOR.EncodeUint64(uint64(v)) } -// Encode encodes SomeStorable as +var _ atree.ContainerStorable = &SomeStorable{} + +func (s SomeStorable) Encode(e *atree.Encoder) error { + nonSomeStorable, nestedLevels := s.nonSomeStorable() + if nestedLevels == 1 { + return s.encode(e) + } + return s.encodeMultipleNestedLevels(e, nestedLevels, nonSomeStorable) +} + +// encode encodes SomeStorable with nested levels = 1 as // // cbor.Tag{ // Number: CBORTagSomeValue, // Content: Value(v.Value), // } -func (s SomeStorable) Encode(e *atree.Encoder) error { +func (s SomeStorable) encode(e *atree.Encoder) error { // NOTE: when updating, also update SomeStorable.ByteSize err := e.CBOR.EncodeRawBytes([]byte{ // tag number @@ -699,6 +709,41 @@ func (s SomeStorable) Encode(e *atree.Encoder) error { return s.Storable.Encode(e) } +const ( + someStorableWithMultipleNestedlevelsArraySize = 1 + someStorableWithMultipleNestedLevelsArrayCount = 2 +) + +// encodeMultipleNestedLevels encodes SomeStorable with nested levels > 1 as +// +// cbor.Tag{ +// Number: CBORTagSomeValueWithNestedLevels, +// Content: CBORArray[nested_levels, innermsot_value], +// } +func (s SomeStorable) encodeMultipleNestedLevels( + e *atree.Encoder, + levels uint64, + nonSomeStorable atree.Storable, +) error { + // NOTE: when updating, also update SomeStorable.ByteSize + err := e.CBOR.EncodeRawBytes([]byte{ + // tag number + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + }) + if err != nil { + return err + } + + err = e.CBOR.EncodeUint64(levels) + if err != nil { + return err + } + + return nonSomeStorable.Encode(e) +} + // Encode encodes AddressValue as // // cbor.Tag{ @@ -1601,6 +1646,19 @@ var _ atree.TypeInfo = compositeTypeInfo{} const encodedCompositeTypeInfoLength = 3 +func (c compositeTypeInfo) IsComposite() bool { + return true +} + +func (c compositeTypeInfo) Identifier() string { + return string(c.location.TypeID(nil, c.qualifiedIdentifier)) +} + +func (c compositeTypeInfo) Copy() atree.TypeInfo { + // Return c as is because c is a value type. + return c +} + func (c compositeTypeInfo) Encode(e *cbor.StreamEncoder) error { err := e.EncodeRawBytes([]byte{ // tag number @@ -1645,4 +1703,16 @@ func (e EmptyTypeInfo) Encode(encoder *cbor.StreamEncoder) error { return encoder.EncodeNil() } +func (e EmptyTypeInfo) IsComposite() bool { + return false +} + +func (e EmptyTypeInfo) Identifier() string { + return "" +} + +func (e EmptyTypeInfo) Copy() atree.TypeInfo { + return e +} + var emptyTypeInfo atree.TypeInfo = EmptyTypeInfo{} diff --git a/runtime/interpreter/encoding_test.go b/runtime/interpreter/encoding_test.go index 91b8764404..99e1c83d55 100644 --- a/runtime/interpreter/encoding_test.go +++ b/runtime/interpreter/encoding_test.go @@ -19,6 +19,7 @@ package interpreter_test import ( + "bytes" "math" "math/big" "strings" @@ -78,7 +79,7 @@ func testEncodeDecode(t *testing.T, test encodeDecodeTest) { } var err error - encoded, err = atree.Encode(test.storable, CBOREncMode) + encoded, err = encodeStorable(test.storable, CBOREncMode) require.NoError(t, err) if test.encoded != nil { @@ -89,7 +90,7 @@ func testEncodeDecode(t *testing.T, test encodeDecodeTest) { } decoder := CBORDecMode.NewByteStreamDecoder(encoded) - decodedStorable, err := DecodeStorable(decoder, test.slabStorageID, nil) + decodedStorable, err := DecodeStorable(decoder, test.slabStorageID, nil, nil) if test.invalid { require.Error(t, err) @@ -126,6 +127,24 @@ func testEncodeDecode(t *testing.T, test encodeDecodeTest) { } } +// encodeStorable wraps storable.Encode() +func encodeStorable(storable atree.Storable, encMode cbor.EncMode) ([]byte, error) { + var buf bytes.Buffer + enc := atree.NewEncoder(&buf, encMode) + + err := storable.Encode(enc) + if err != nil { + return nil, err + } + + err = enc.CBOR.Flush() + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + func TestEncodeDecodeNilValue(t *testing.T) { t.Parallel() @@ -389,6 +408,8 @@ func TestEncodeDecodeUint64AtreeValue(t *testing.T) { func TestEncodeDecodeArray(t *testing.T) { + t.Skip("skipping ArrayValue encoding and decoding test because it involves implementation details in atree repo") + t.Parallel() t.Run("empty", func(t *testing.T) { @@ -458,6 +479,8 @@ func TestEncodeDecodeArray(t *testing.T) { func TestEncodeDecodeComposite(t *testing.T) { + t.Skip("skipping CompositeValue encoding and decoding test because it involves implementation details in atree repo") + t.Parallel() t.Run("empty structure, string location, qualified identifier", func(t *testing.T) { @@ -2746,7 +2769,7 @@ func TestEncodeDecodeSomeValue(t *testing.T) { t.Parallel() - t.Run("nil", func(t *testing.T) { + t.Run("SomeValue{nil}", func(t *testing.T) { t.Parallel() @@ -2763,7 +2786,114 @@ func TestEncodeDecodeSomeValue(t *testing.T) { ) }) - t.Run("string", func(t *testing.T) { + t.Run("SomeValue{SomeValue{nil}}", func(t *testing.T) { + + t.Parallel() + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying(Nil)), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, + // null + 0xf6, + }, + }, + ) + }) + + t.Run("SomeValue{SomeValue{SomeValue{nil}}}", func(t *testing.T) { + + t.Parallel() + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + Nil))), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 3 + 0x03, + // null + 0xf6, + }, + }, + ) + }) + + t.Run("SomeValue{bool}", func(t *testing.T) { + t.Parallel() + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying(TrueValue), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValue, + // true + 0xf5, + }, + }, + ) + }) + + t.Run("SomeValue{SomeValue{bool}}", func(t *testing.T) { + t.Parallel() + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + TrueValue)), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, + // true + 0xf5, + }, + }, + ) + }) + + t.Run("SomeValue{SomeValue{SomeValue{bool}}}", func(t *testing.T) { + t.Parallel() + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + TrueValue))), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 3 + 0x03, + // true + 0xf5, + }, + }, + ) + }) + + t.Run("SomeValue{string}", func(t *testing.T) { t.Parallel() expectedString := NewUnmeteredStringValue("test") @@ -2774,10 +2904,65 @@ func TestEncodeDecodeSomeValue(t *testing.T) { encoded: []byte{ // tag 0xd8, CBORTagSomeValue, + // tag + 0xd8, CBORTagStringValue, + // UTF-8 string, length 4 + 0x64, + // t, e, s, t + 0x74, 0x65, 0x73, 0x74, + }, + }, + ) + }) + + t.Run("SomeValue{SomeValue{string}}", func(t *testing.T) { + t.Parallel() + + expectedString := NewUnmeteredStringValue("test") + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + expectedString)), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, // tag 0xd8, CBORTagStringValue, + // UTF-8 string, length 4 + 0x64, + // t, e, s, t + 0x74, 0x65, 0x73, 0x74, + }, + }, + ) + }) + t.Run("SomeValue{SomeValue{SomeValue{string}}}", func(t *testing.T) { + t.Parallel() + + expectedString := NewUnmeteredStringValue("test") + + testEncodeDecode(t, + encodeDecodeTest{ + value: NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + expectedString))), + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 3 + 0x03, + // tag + 0xd8, CBORTagStringValue, // UTF-8 string, length 4 0x64, // t, e, s, t @@ -2787,28 +2972,106 @@ func TestEncodeDecodeSomeValue(t *testing.T) { ) }) - t.Run("bool", func(t *testing.T) { + t.Run("SomeValue{large_inlined_string}", func(t *testing.T) { + t.Parallel() + const ( + cborTagSize = 2 + ) + + var str *StringValue + maxInlineElementSize := uint64(64) // use a small max inline size for testing + for i := uint64(0); i < maxInlineElementSize; i++ { + str = NewUnmeteredStringValue(strings.Repeat("x", int(maxInlineElementSize-i))) + size, err := StorableSize(str) + require.NoError(t, err) + if uint64(size) < maxInlineElementSize-cborTagSize { + break + } + } + + expected := NewUnmeteredSomeValueNonCopying(str) + testEncodeDecode(t, encodeDecodeTest{ - value: NewUnmeteredSomeValueNonCopying(TrueValue), + value: expected, + maxInlineElementSize: maxInlineElementSize, encoded: []byte{ // tag 0xd8, CBORTagSomeValue, - // true - 0xf5, + // tag + 0xd8, CBORTagStringValue, + // text string (57 bytes) + 0x78, 0x39, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, }, }, ) }) - t.Run("larger than max inline size", func(t *testing.T) { + t.Run("SomeValue{SomeValue{large_inlined_string}}", func(t *testing.T) { + + t.Parallel() + + const ( + cborTagSize = 2 + arraySize = 1 + nestedLevelsSize = 1 + ) + + var str *StringValue + maxInlineElementSize := uint64(64) // use a small max inline size for testing + for i := uint64(0); i < maxInlineElementSize; i++ { + str = NewUnmeteredStringValue(strings.Repeat("x", int(maxInlineElementSize-i))) + size, err := StorableSize(str) + require.NoError(t, err) + if uint64(size) < maxInlineElementSize-cborTagSize-arraySize-nestedLevelsSize { + break + } + } + + expected := NewUnmeteredSomeValueNonCopying(NewUnmeteredSomeValueNonCopying(str)) + + testEncodeDecode(t, + encodeDecodeTest{ + value: expected, + maxInlineElementSize: maxInlineElementSize, + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, + // tag + 0xd8, CBORTagStringValue, + // text string (55 bytes) + 0x78, 0x37, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, + }, + }, + ) + }) + + t.Run("SomeValue{large_string_stored_separately}", func(t *testing.T) { t.Parallel() - // Generate a strings that has an encoding size just below the max inline element size. - // It will not get inlined, but the outer value will + // Generate a string that has an encoding size just above the max inline element size var str *StringValue maxInlineElementSize := atree.MaxInlineArrayElementSize() @@ -2816,7 +3079,7 @@ func TestEncodeDecodeSomeValue(t *testing.T) { str = NewUnmeteredStringValue(strings.Repeat("x", int(maxInlineElementSize-i))) size, err := StorableSize(str) require.NoError(t, err) - if uint64(size) == maxInlineElementSize-1 { + if uint64(size) == maxInlineElementSize+1 { break } } @@ -2829,8 +3092,9 @@ func TestEncodeDecodeSomeValue(t *testing.T) { maxInlineElementSize: maxInlineElementSize, encoded: []byte{ // tag + 0xd8, CBORTagSomeValue, + // value 0xd8, atree.CBORTagSlabID, - // storage ID 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, }, @@ -2838,7 +3102,7 @@ func TestEncodeDecodeSomeValue(t *testing.T) { ) }) - t.Run("inner path stored separately", func(t *testing.T) { + t.Run("SomeValue{SomeValue{large_string_stored_separately}}", func(t *testing.T) { t.Parallel() @@ -2855,10 +3119,162 @@ func TestEncodeDecodeSomeValue(t *testing.T) { } } - expected := NewUnmeteredSomeValueNonCopying(str) + expected := NewUnmeteredSomeValueNonCopying(NewUnmeteredSomeValueNonCopying(str)) testEncodeDecode(t, encodeDecodeTest{ + value: expected, + maxInlineElementSize: maxInlineElementSize, + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels + 0x02, + // value + 0xd8, atree.CBORTagSlabID, + // storage ID + 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + }, + }, + ) + }) + + t.Run("SomeValue{inlined_array}", func(t *testing.T) { + t.Parallel() + + inter := newTestInterpreter(t) + + expected := NewUnmeteredSomeValueNonCopying( + NewArrayValue( + inter, + EmptyLocationRange, + ConstantSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + Size: 0, + }, + testOwner, + )) + + expectedEncoded := []byte{ + // tag + 0xd8, CBORTagSomeValue, + + // tag + 0xd8, atree.CBORTagInlinedArray, + + // array of 3 elements + 0x83, + + // extra data index (extra data is encoded in slab header, not here) + 0x18, 0x00, + + // inlined array slab index + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + + // inlined array of 0 elements + 0x99, 0x00, 0x00, + } + + // Only test encoding here because decoding requires extra data section which is encoded in slab header. + + storage := newUnmeteredInMemoryStorage() + storable, err := expected.Storable( + storage, + atree.Address(testOwner), + atree.MaxInlineArrayElementSize(), + ) + require.NoError(t, err) + + encoded, err := encodeStorable(storable, CBOREncMode) + require.NoError(t, err) + AssertEqualWithDiff(t, expectedEncoded, encoded) + }) + + t.Run("SomeValue{SomeValue{inlined_array}}", func(t *testing.T) { + t.Parallel() + + inter := newTestInterpreter(t) + + expected := NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + NewArrayValue( + inter, + EmptyLocationRange, + ConstantSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + Size: 0, + }, + testOwner, + ))) + + expectedEncoded := []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + + // array of 2 elements + 0x82, + + // nested levels: 2 + 0x02, + + // tag + 0xd8, atree.CBORTagInlinedArray, + + // array of 3 elements + 0x83, + + // extra data index (extra data section is encoded in slab header, not here) + 0x18, 0x00, + + // inlined array slab index + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + + // inlined array of 0 elements + 0x99, 0x00, 0x00, + } + + // Only test encoding here because decoding requires extra data section which is encoded in slab header. + + storage := newUnmeteredInMemoryStorage() + storable, err := expected.Storable( + storage, + atree.Address(testOwner), + atree.MaxInlineArrayElementSize(), + ) + require.NoError(t, err) + + encoded, err := encodeStorable(storable, CBOREncMode) + require.NoError(t, err) + AssertEqualWithDiff(t, expectedEncoded, encoded) + }) + + t.Run("SomeValue{large_array_stored_separately}", func(t *testing.T) { + t.Parallel() + + inter := newTestInterpreter(t) + + maxInlineElementSize := atree.MaxInlineArrayElementSize() + + expectedArray := + NewArrayValue( + inter, + EmptyLocationRange, + ConstantSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + Size: 0, + }, + testOwner, + NewUnmeteredStringValue(strings.Repeat("a", int(maxInlineElementSize/2))), + NewUnmeteredStringValue(strings.Repeat("b", int(maxInlineElementSize/2))), + ) + + expected := NewUnmeteredSomeValueNonCopying(expectedArray) + + testEncodeDecode(t, + encodeDecodeTest{ + storage: inter.Storage(), value: expected, maxInlineElementSize: maxInlineElementSize, encoded: []byte{ @@ -2872,6 +3288,121 @@ func TestEncodeDecodeSomeValue(t *testing.T) { }, ) }) + + t.Run("SomeValue{SomeValue{large_array_stored_separately}}", func(t *testing.T) { + t.Parallel() + + inter := newTestInterpreter(t) + + maxInlineElementSize := atree.MaxInlineArrayElementSize() + + expectedArray := + NewArrayValue( + inter, + EmptyLocationRange, + ConstantSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + Size: 0, + }, + testOwner, + NewUnmeteredStringValue(strings.Repeat("a", int(maxInlineElementSize/2))), + NewUnmeteredStringValue(strings.Repeat("b", int(maxInlineElementSize/2))), + ) + + expected := NewUnmeteredSomeValueNonCopying(NewUnmeteredSomeValueNonCopying(expectedArray)) + + testEncodeDecode(t, + encodeDecodeTest{ + storage: inter.Storage(), + value: expected, + maxInlineElementSize: maxInlineElementSize, + encoded: []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, + // value + 0xd8, atree.CBORTagSlabID, + // storage ID + 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + }, + }, + ) + }) + + t.Run("SomeValue{SomeValue{[SomeValue{SomeValue{SomeValue{1}}}]}}", func(t *testing.T) { + t.Parallel() + + inter := newTestInterpreter(t) + + // [ SomeValue { SomeValue { SomeValue { 1 } } } ] + expectedArray := + NewArrayValue( + inter, + EmptyLocationRange, + ConstantSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + Size: 0, + }, + testOwner, + NewUnmeteredSomeValueNonCopying( // SomeValue { SomeValue { SomeValue { 1 } } } + NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + UInt64Value(1)))), + ) + + // SomeValue { SomeValue { [ SomeValue { SomeValue { SomeValue { 1 } } } ] } } + expected := NewUnmeteredSomeValueNonCopying( + NewUnmeteredSomeValueNonCopying( + expectedArray, + )) + + expectedEncoded := []byte{ + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 2 + 0x02, + // value + 0xd8, atree.CBORTagInlinedArray, + // array of 3 elements + 0x83, + // extra data index (extra data section is encoded in slab header, not here) + 0x18, 0x00, + // inlined array slab index + 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // inlined array of 1 elements + 0x99, 0x00, 0x01, + // element 0: SomeValue { SomeValue { SomeValue { 1 } } } + // tag + 0xd8, CBORTagSomeValueWithNestedLevels, + // array of 2 elements + 0x82, + // nested levels: 3 + 0x03, + // tag + 0xd8, CBORTagUInt64Value, + // integer 1 + 0x1, + } + + // Only test encoding here because decoding requires extra data section which is encoded in slab header. + + storage := newUnmeteredInMemoryStorage() + storable, err := expected.Storable( + storage, + atree.Address(testOwner), + atree.MaxInlineArrayElementSize(), + ) + require.NoError(t, err) + + encoded, err := encodeStorable(storable, CBOREncMode) + require.NoError(t, err) + AssertEqualWithDiff(t, expectedEncoded, encoded) + }) } func TestEncodeDecodeFix64Value(t *testing.T) { diff --git a/runtime/interpreter/interpreter.go b/runtime/interpreter/interpreter.go index b7646d431d..3e3b349127 100644 --- a/runtime/interpreter/interpreter.go +++ b/runtime/interpreter/interpreter.go @@ -831,14 +831,14 @@ func (interpreter *Interpreter) resultValue(returnValue Value, returnType sema.T optionalType.Type, ) - interpreter.maybeTrackReferencedResourceKindedValue(returnValue.value) + interpreter.MaybeTrackReferencedResourceKindedValue(returnValue.value) return NewSomeValueNonCopying(interpreter, innerValue) case NilValue: return NilValue{} } } - interpreter.maybeTrackReferencedResourceKindedValue(returnValue) + interpreter.MaybeTrackReferencedResourceKindedValue(returnValue) return NewEphemeralReferenceValue(interpreter, false, returnValue, returnType) } @@ -1706,6 +1706,7 @@ func (interpreter *Interpreter) transferAndConvert( false, nil, nil, + true, // value is standalone. ) result := interpreter.ConvertAndBox( @@ -3924,6 +3925,7 @@ func (interpreter *Interpreter) authAccountSaveFunction(addressValue AddressValu true, nil, nil, + true, // value is standalone because it is from invocation.Arguments[0]. ) // Write new value @@ -4048,6 +4050,7 @@ func (interpreter *Interpreter) authAccountReadFunction(addressValue AddressValu false, nil, nil, + false, // value is an element in storage map because it is from "ReadStored". ) // Remove the value from storage, @@ -5036,6 +5039,10 @@ func (interpreter *Interpreter) maybeValidateAtreeValue(v atree.Value) { if config.AtreeValueValidationEnabled { interpreter.ValidateAtreeValue(v) } +} + +func (interpreter *Interpreter) maybeValidateAtreeStorage() { + config := interpreter.SharedState.Config if config.AtreeStorageValidationEnabled { err := config.Storage.CheckHealth() @@ -5120,14 +5127,16 @@ func (interpreter *Interpreter) ValidateAtreeValue(value atree.Value) { } } + atreeInliningEnabled := true + switch value := value.(type) { case *atree.Array: - err := atree.ValidArray(value, value.Type(), tic, hip) + err := atree.VerifyArray(value, value.Address(), value.Type(), tic, hip, atreeInliningEnabled) if err != nil { panic(errors.NewExternalError(err)) } - err = atree.ValidArraySerialization( + err = atree.VerifyArraySerialization( value, CBORDecMode, CBOREncMode, @@ -5148,12 +5157,12 @@ func (interpreter *Interpreter) ValidateAtreeValue(value atree.Value) { } case *atree.OrderedMap: - err := atree.ValidMap(value, value.Type(), tic, hip) + err := atree.VerifyMap(value, value.Address(), value.Type(), tic, hip, atreeInliningEnabled) if err != nil { panic(errors.NewExternalError(err)) } - err = atree.ValidMapSerialization( + err = atree.VerifyMapSerialization( value, CBORDecMode, CBOREncMode, @@ -5175,7 +5184,7 @@ func (interpreter *Interpreter) ValidateAtreeValue(value atree.Value) { } } -func (interpreter *Interpreter) maybeTrackReferencedResourceKindedValue(value Value) { +func (interpreter *Interpreter) MaybeTrackReferencedResourceKindedValue(value Value) { if value, ok := value.(ReferenceTrackedResourceKindedValue); ok { interpreter.trackReferencedResourceKindedValue(value.ValueID(), value) } @@ -5330,11 +5339,12 @@ func (interpreter *Interpreter) MeterMemory(usage common.MemoryUsage) error { func (interpreter *Interpreter) DecodeStorable( decoder *cbor.StreamDecoder, slabID atree.SlabID, + inlinedExtraData []atree.ExtraData, ) ( atree.Storable, error, ) { - return DecodeStorable(decoder, slabID, interpreter) + return DecodeStorable(decoder, slabID, inlinedExtraData, interpreter) } func (interpreter *Interpreter) DecodeTypeInfo(decoder *cbor.StreamDecoder) (atree.TypeInfo, error) { diff --git a/runtime/interpreter/interpreter_expression.go b/runtime/interpreter/interpreter_expression.go index 82e866d4c0..0bc51b1b6d 100644 --- a/runtime/interpreter/interpreter_expression.go +++ b/runtime/interpreter/interpreter_expression.go @@ -1183,7 +1183,7 @@ func (interpreter *Interpreter) VisitReferenceExpression(referenceExpression *as result := interpreter.evalExpression(referenceExpression.Expression) - interpreter.maybeTrackReferencedResourceKindedValue(result) + interpreter.MaybeTrackReferencedResourceKindedValue(result) // There are four potential cases: // 1) Target type is optional, actual value is also optional (nil/SomeValue) @@ -1211,7 +1211,7 @@ func (interpreter *Interpreter) VisitReferenceExpression(referenceExpression *as } innerValue := result.InnerValue(interpreter, locationRange) - interpreter.maybeTrackReferencedResourceKindedValue(innerValue) + interpreter.MaybeTrackReferencedResourceKindedValue(innerValue) return NewSomeValueNonCopying( interpreter, @@ -1258,7 +1258,7 @@ func (interpreter *Interpreter) VisitReferenceExpression(referenceExpression *as HasPosition: referenceExpression.Expression, } innerValue := someValue.InnerValue(interpreter, locationRange) - interpreter.maybeTrackReferencedResourceKindedValue(innerValue) + interpreter.MaybeTrackReferencedResourceKindedValue(innerValue) } // Case (4): target type is non-optional, actual value is also non-optional @@ -1362,6 +1362,7 @@ func (interpreter *Interpreter) VisitAttachExpression(attachExpression *ast.Atta false, nil, nil, + true, // base is standalone. ).(*CompositeValue) // we enforce this in the checker diff --git a/runtime/interpreter/interpreter_invocation.go b/runtime/interpreter/interpreter_invocation.go index d0d18d39ea..7311517e49 100644 --- a/runtime/interpreter/interpreter_invocation.go +++ b/runtime/interpreter/interpreter_invocation.go @@ -101,6 +101,7 @@ func (interpreter *Interpreter) invokeFunctionValue( false, nil, nil, + true, // argument is standalone. ) } } diff --git a/runtime/interpreter/interpreter_statement.go b/runtime/interpreter/interpreter_statement.go index 1a0a12fa1f..48f74239c1 100644 --- a/runtime/interpreter/interpreter_statement.go +++ b/runtime/interpreter/interpreter_statement.go @@ -283,6 +283,7 @@ func (interpreter *Interpreter) VisitForStatement(statement *ast.ForStatement) S false, nil, nil, + true, // value is standalone ) iterable, ok := transferredValue.(IterableValue) diff --git a/runtime/interpreter/interpreter_tracing_test.go b/runtime/interpreter/interpreter_tracing_test.go index d36bbcee9b..5176534a57 100644 --- a/runtime/interpreter/interpreter_tracing_test.go +++ b/runtime/interpreter/interpreter_tracing_test.go @@ -79,7 +79,7 @@ func TestInterpreterTracing(t *testing.T) { cloned := array.Clone(inter) require.NotNil(t, cloned) - cloned.DeepRemove(inter) + cloned.DeepRemove(inter, true) require.Equal(t, len(traceOps), 2) require.Equal(t, traceOps[1], "array.deepRemove") @@ -109,7 +109,7 @@ func TestInterpreterTracing(t *testing.T) { cloned := dict.Clone(inter) require.NotNil(t, cloned) - cloned.DeepRemove(inter) + cloned.DeepRemove(inter, true) require.Equal(t, len(traceOps), 2) require.Equal(t, traceOps[1], "dictionary.deepRemove") @@ -132,7 +132,7 @@ func TestInterpreterTracing(t *testing.T) { cloned := value.Clone(inter) require.NotNil(t, cloned) - cloned.DeepRemove(inter) + cloned.DeepRemove(inter, true) require.Equal(t, len(traceOps), 2) require.Equal(t, traceOps[1], "composite.deepRemove") diff --git a/runtime/interpreter/simplecompositevalue.go b/runtime/interpreter/simplecompositevalue.go index a903d551ac..4cdc93fcca 100644 --- a/runtime/interpreter/simplecompositevalue.go +++ b/runtime/interpreter/simplecompositevalue.go @@ -71,7 +71,7 @@ func NewSimpleCompositeValue( func (*SimpleCompositeValue) isValue() {} -func (v *SimpleCompositeValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *SimpleCompositeValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitSimpleCompositeValue(interpreter, v) } @@ -90,7 +90,7 @@ func (v *SimpleCompositeValue) ForEachField( // Walk iterates over all field values of the composite value. // It does NOT walk the computed fields and functions! -func (v *SimpleCompositeValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *SimpleCompositeValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { v.ForEachField(func(_ string, fieldValue Value) (resume bool) { walkChild(fieldValue) @@ -262,6 +262,7 @@ func (v *SimpleCompositeValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { // TODO: actually not needed, value is not storable if remove { @@ -291,6 +292,6 @@ func (v *SimpleCompositeValue) Clone(interpreter *Interpreter) Value { } } -func (v *SimpleCompositeValue) DeepRemove(_ *Interpreter) { +func (v *SimpleCompositeValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/statictype.go b/runtime/interpreter/statictype.go index adb270488e..d3b28b7e65 100644 --- a/runtime/interpreter/statictype.go +++ b/runtime/interpreter/statictype.go @@ -233,6 +233,7 @@ type ArrayStaticType interface { StaticType isArrayStaticType() ElementType() StaticType + atree.TypeInfo } // VariableSizedStaticType @@ -255,6 +256,19 @@ func NewVariableSizedStaticType( } } +func (t VariableSizedStaticType) IsComposite() bool { + return false +} + +func (t VariableSizedStaticType) Copy() atree.TypeInfo { + // VariableSizedStaticType is never mutated, return a shallow copy + return t +} + +func (t VariableSizedStaticType) Identifier() string { + return string(t.ID()) +} + func (VariableSizedStaticType) isStaticType() {} func (VariableSizedStaticType) elementSize() uint { @@ -314,6 +328,19 @@ func NewConstantSizedStaticType( } } +func (t ConstantSizedStaticType) IsComposite() bool { + return false +} + +func (t ConstantSizedStaticType) Copy() atree.TypeInfo { + // ConstantSizedStaticType is never mutated, return a shallow copy + return t +} + +func (t ConstantSizedStaticType) Identifier() string { + return string(t.ID()) +} + func (ConstantSizedStaticType) isStaticType() {} func (ConstantSizedStaticType) elementSize() uint { @@ -380,6 +407,19 @@ func NewDictionaryStaticType( } } +func (t DictionaryStaticType) IsComposite() bool { + return false +} + +func (t DictionaryStaticType) Copy() atree.TypeInfo { + // DictionaryStaticType is never mutated, return a shallow copy + return t +} + +func (t DictionaryStaticType) Identifier() string { + return string(t.ID()) +} + func (DictionaryStaticType) isStaticType() {} func (DictionaryStaticType) elementSize() uint { @@ -788,15 +828,17 @@ func ConvertSemaArrayTypeToStaticArrayType( ) ArrayStaticType { switch t := t.(type) { case *sema.VariableSizedType: - return VariableSizedStaticType{ - Type: ConvertSemaToStaticType(memoryGauge, t.Type), - } + return NewVariableSizedStaticType( + memoryGauge, + ConvertSemaToStaticType(memoryGauge, t.Type), + ) case *sema.ConstantSizedType: - return ConstantSizedStaticType{ - Type: ConvertSemaToStaticType(memoryGauge, t.Type), - Size: t.Size, - } + return NewConstantSizedStaticType( + memoryGauge, + ConvertSemaToStaticType(memoryGauge, t.Type), + t.Size, + ) default: panic(errors.NewUnreachableError()) diff --git a/runtime/interpreter/storage.go b/runtime/interpreter/storage.go index 666a530274..87f8fef8d4 100644 --- a/runtime/interpreter/storage.go +++ b/runtime/interpreter/storage.go @@ -121,8 +121,12 @@ type InMemoryStorage struct { var _ Storage = InMemoryStorage{} func NewInMemoryStorage(memoryGauge common.MemoryGauge) InMemoryStorage { - decodeStorable := func(decoder *cbor.StreamDecoder, storableSlabStorageID atree.SlabID) (atree.Storable, error) { - return DecodeStorable(decoder, storableSlabStorageID, memoryGauge) + decodeStorable := func( + decoder *cbor.StreamDecoder, + storableSlabStorageID atree.SlabID, + inlinedExtraData []atree.ExtraData, + ) (atree.Storable, error) { + return DecodeStorable(decoder, storableSlabStorageID, inlinedExtraData, memoryGauge) } decodeTypeInfo := func(decoder *cbor.StreamDecoder) (atree.TypeInfo, error) { diff --git a/runtime/interpreter/storage_test.go b/runtime/interpreter/storage_test.go index 73a2d1d56c..8e2da5f450 100644 --- a/runtime/interpreter/storage_test.go +++ b/runtime/interpreter/storage_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/sema" "github.com/onflow/cadence/runtime/common" @@ -148,8 +149,9 @@ func TestArrayStorage(t *testing.T) { require.True(t, bool(value.Contains(inter, EmptyLocationRange, element))) - // array + original composite element + new copy of composite element - require.Equal(t, 3, storage.BasicSlabStorage.Count()) + // array + new copy of composite element + // NOTE: original composite value is inlined in parent array. + require.Equal(t, 2, storage.BasicSlabStorage.Count()) retrievedStorable, ok, err := storage.BasicSlabStorage.Retrieve(value.SlabID()) require.NoError(t, err) @@ -199,8 +201,9 @@ func TestArrayStorage(t *testing.T) { require.NotEqual(t, atree.SlabIDUndefined, value.SlabID()) - // array + original composite element + new copy of composite element - require.Equal(t, 3, storage.BasicSlabStorage.Count()) + // array + new copy of composite element + // NOTE: original composite value is inlined in parent array. + require.Equal(t, 2, storage.BasicSlabStorage.Count()) _, ok, err := storage.BasicSlabStorage.Retrieve(value.SlabID()) require.NoError(t, err) @@ -436,60 +439,483 @@ func TestDictionaryStorage(t *testing.T) { }) } -func TestInterpretStorageOverwriteAndRemove(t *testing.T) { +func TestStorageOverwriteAndRemove(t *testing.T) { t.Parallel() - storage := newUnmeteredInMemoryStorage() + t.Run("overwrite inlined value with inlined value", func(t *testing.T) { - inter, err := NewInterpreter( - nil, - common.AddressLocation{}, - &Config{Storage: storage}, - ) - require.NoError(t, err) + storage := newUnmeteredInMemoryStorage() - address := common.ZeroAddress + inter, err := NewInterpreter( + nil, + common.AddressLocation{}, + &Config{Storage: storage}, + ) + require.NoError(t, err) - array1 := NewArrayValue( - inter, - EmptyLocationRange, - VariableSizedStaticType{ - Type: PrimitiveStaticTypeAnyStruct, - }, - address, - NewUnmeteredStringValue("first"), - ) + address := common.ZeroAddress - const storageMapKey = StringStorageMapKey("test") + array1 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + address, + NewUnmeteredStringValue("first"), + ) - storageMap := storage.GetStorageMap(address, "storage", true) - storageMap.WriteValue(inter, storageMapKey, array1) + const storageMapKey = StringStorageMapKey("test") - // Overwriting delete any existing child slabs + storageMap := storage.GetStorageMap(address, "storage", true) + storageMap.WriteValue(inter, storageMapKey, array1) - array2 := NewArrayValue( - inter, - EmptyLocationRange, - VariableSizedStaticType{ - Type: PrimitiveStaticTypeAnyStruct, - }, - address, - NewUnmeteredStringValue("second"), - ) + // Overwriting delete any existing child slabs + + array2 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + address, + NewUnmeteredStringValue("second"), + ) - storageMap.WriteValue(inter, storageMapKey, array2) + storageMap.WriteValue(inter, storageMapKey, array2) - // 2: - // - storage map (atree ordered map) - // - array (atree array) - assert.Len(t, storage.Slabs, 2) + // 1: + // - storage map (atree ordered map) + // NOTE: array (atree array) is inlined in storage map + assert.Len(t, storage.Slabs, 1) - // Writing nil is deletion and should delete any child slabs + // Writing nil is deletion and should delete any child slabs - storageMap.WriteValue(inter, storageMapKey, nil) + storageMap.WriteValue(inter, storageMapKey, nil) + + // 1: + // - storage map (atree ordered map) + assert.Len(t, storage.Slabs, 1) + }) + + // TODO: add subtests to + // - overwrite inlined value with not inlined value + // - overwrite not inlined value with not inlined value + // - overwrite not inlined value with inlined value +} + +func TestNestedContainerMutationAfterMove(t *testing.T) { + + t.Parallel() + + testStructType := &sema.CompositeType{ + Location: TestLocation, + Identifier: "TestStruct", + Kind: common.CompositeKindStructure, + Members: &sema.StringMemberOrderedMap{}, + } + + testResourceType := &sema.CompositeType{ + Location: TestLocation, + Identifier: "TestResource", + Kind: common.CompositeKindStructure, + Members: &sema.StringMemberOrderedMap{}, + } + + const fieldName = "test" + + for _, testCompositeType := range []*sema.CompositeType{ + testStructType, + testResourceType, + } { + fieldMember := sema.NewFieldMember( + nil, + testCompositeType, + ast.AccessPublic, + ast.VariableKindVariable, + fieldName, + sema.UInt8Type, + "", + ) + testCompositeType.Members.Set(fieldName, fieldMember) + } + + importLocationHandlerFunc := func(inter *Interpreter, location common.Location) Import { + elaboration := sema.NewElaboration(nil) + elaboration.SetCompositeType( + testStructType.ID(), + testStructType, + ) + elaboration.SetCompositeType( + testResourceType.ID(), + testResourceType, + ) + return VirtualImport{Elaboration: elaboration} + } + + t.Run("struct, move from array to array", func(t *testing.T) { + + t.Parallel() + + storage := newUnmeteredInMemoryStorage() + + inter, err := NewInterpreter( + nil, + common.AddressLocation{}, + &Config{ + Storage: storage, + ImportLocationHandler: importLocationHandlerFunc, + }, + ) + require.NoError(t, err) + + containerValue1 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + ) + + containerValue2 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + ) + + newChildValue := func(value uint8) *CompositeValue { + return NewCompositeValue( + inter, + EmptyLocationRange, + TestLocation, + "TestStruct", + common.CompositeKindStructure, + []CompositeField{ + { + Name: fieldName, + Value: NewUnmeteredUInt8Value(value), + }, + }, + common.ZeroAddress, + ) + } + + childValue1 := newChildValue(0) + + require.Equal(t, "[]", containerValue1.String()) + require.Equal(t, "[]", containerValue2.String()) + + containerValue1.Append(inter, EmptyLocationRange, NewUnmeteredUInt8Value(1)) + containerValue2.Append(inter, EmptyLocationRange, NewUnmeteredUInt8Value(2)) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + + require.Equal(t, "S.test.TestStruct(test: 0)", childValue1.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(3)) + + require.Equal(t, "S.test.TestStruct(test: 3)", childValue1.String()) + + childValue2 := childValue1.Transfer( + inter, + EmptyLocationRange, + atree.Address{}, + false, + nil, + map[atree.ValueID]struct{}{}, + true, // childValue1 is standalone before being inserted into containerValue1. + ).(*CompositeValue) + + containerValue1.Append(inter, EmptyLocationRange, childValue1) + // Append invalidated, get again + childValue1 = containerValue1.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestStruct(test: 3)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 3)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 3)", childValue2.String()) + + childValue2.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(4)) + + require.Equal(t, "[1, S.test.TestStruct(test: 3)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 3)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(5)) + + require.Equal(t, "[1, S.test.TestStruct(test: 5)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + + childValue3 := containerValue1.Remove(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + // TODO: fix + require.Equal(t, "S.test.TestStruct()", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(6)) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 6)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + + childValue4 := newChildValue(7) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 6)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 7)", childValue4.String()) + + containerValue1.Append(inter, EmptyLocationRange, childValue4) + // Append invalidated, get again + childValue4 = containerValue1.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestStruct(test: 7)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 6)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 7)", childValue4.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(8)) + + require.Equal(t, "[1, S.test.TestStruct(test: 7)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 8)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 7)", childValue4.String()) + + childValue4.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(9)) + + require.Equal(t, "[1, S.test.TestStruct(test: 9)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 8)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 9)", childValue4.String()) + + containerValue2.Append(inter, EmptyLocationRange, childValue3) + // Append invalidated, get again + childValue3 = containerValue2.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestStruct(test: 9)]", containerValue1.String()) + require.Equal(t, "[2, S.test.TestStruct(test: 5)]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 8)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 5)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 9)", childValue4.String()) + + childValue3.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(10)) + + require.Equal(t, "[1, S.test.TestStruct(test: 9)]", containerValue1.String()) + require.Equal(t, "[2, S.test.TestStruct(test: 10)]", containerValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 8)", childValue1.String()) + require.Equal(t, "S.test.TestStruct(test: 4)", childValue2.String()) + require.Equal(t, "S.test.TestStruct(test: 10)", childValue3.String()) + require.Equal(t, "S.test.TestStruct(test: 9)", childValue4.String()) + }) + + t.Run("resource, move from array to array", func(t *testing.T) { + + t.Parallel() + + storage := newUnmeteredInMemoryStorage() + + inter, err := NewInterpreter( + nil, + common.AddressLocation{}, + &Config{ + Storage: storage, + ImportLocationHandler: importLocationHandlerFunc, + }, + ) + require.NoError(t, err) + + containerValue1 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + ) + + containerValue2 := NewArrayValue( + inter, + EmptyLocationRange, + VariableSizedStaticType{ + Type: PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + ) + + newChildValue := func(value uint8) *CompositeValue { + return NewCompositeValue( + inter, + EmptyLocationRange, + TestLocation, + "TestResource", + common.CompositeKindResource, + []CompositeField{ + { + Name: fieldName, + Value: NewUnmeteredUInt8Value(value), + }, + }, + common.ZeroAddress, + ) + } + + childValue1 := newChildValue(0) + + require.Equal(t, "[]", containerValue1.String()) + require.Equal(t, "[]", containerValue2.String()) + + containerValue1.Append(inter, EmptyLocationRange, NewUnmeteredUInt8Value(1)) + containerValue2.Append(inter, EmptyLocationRange, NewUnmeteredUInt8Value(2)) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + + require.Equal(t, "S.test.TestResource(test: 0)", childValue1.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(3)) + + require.Equal(t, "S.test.TestResource(test: 3)", childValue1.String()) + + inter.MaybeTrackReferencedResourceKindedValue(childValue1) + ref1 := NewEphemeralReferenceValue(nil, false, childValue1, testResourceType) + + containerValue1.Append(inter, EmptyLocationRange, childValue1) + // Append invalidated, get again + childValue1 = containerValue1.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestResource(test: 3)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 3)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 3)", ref1.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(4)) + + require.Equal(t, "[1, S.test.TestResource(test: 4)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 4)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 4)", ref1.String()) + + ref1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(5)) + + require.Equal(t, "[1, S.test.TestResource(test: 5)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 5)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 5)", ref1.String()) + + childValue2 := containerValue1.Remove(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 5)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 5)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 5)", childValue2.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(6)) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 6)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 6)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 6)", childValue2.String()) + + ref1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(7)) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue2.String()) + + // TODO: rename childValue4 to childValue3 + childValue4 := newChildValue(8) + + require.Equal(t, "[1]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 8)", childValue4.String()) + + containerValue1.Append(inter, EmptyLocationRange, childValue4) + // Append invalidated, get again + childValue4 = containerValue1.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestResource(test: 8)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 7)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 8)", childValue4.String()) + + childValue1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(9)) + + require.Equal(t, "[1, S.test.TestResource(test: 8)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 9)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 9)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 9)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 8)", childValue4.String()) + + ref1.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(10)) + + require.Equal(t, "[1, S.test.TestResource(test: 8)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 8)", childValue4.String()) + + childValue4.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(11)) + + require.Equal(t, "[1, S.test.TestResource(test: 11)]", containerValue1.String()) + require.Equal(t, "[2]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 11)", childValue4.String()) + + containerValue2.Append(inter, EmptyLocationRange, childValue2) + // Append invalidated, get again + childValue2 = containerValue2.Get(inter, EmptyLocationRange, 1).(*CompositeValue) + + require.Equal(t, "[1, S.test.TestResource(test: 11)]", containerValue1.String()) + require.Equal(t, "[2, S.test.TestResource(test: 10)]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 10)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 11)", childValue4.String()) + + childValue2.SetMember(inter, EmptyLocationRange, fieldName, NewUnmeteredUInt8Value(12)) + + require.Equal(t, "[1, S.test.TestResource(test: 11)]", containerValue1.String()) + require.Equal(t, "[2, S.test.TestResource(test: 12)]", containerValue2.String()) + require.Equal(t, "S.test.TestResource(test: 12)", childValue1.String()) + require.Equal(t, "S.test.TestResource(test: 12)", ref1.String()) + require.Equal(t, "S.test.TestResource(test: 12)", childValue2.String()) + require.Equal(t, "S.test.TestResource(test: 11)", childValue4.String()) + }) - // 1: - // - storage map (atree ordered map) - assert.Len(t, storage.Slabs, 1) } diff --git a/runtime/interpreter/storagemap.go b/runtime/interpreter/storagemap.go index 5d7c5a26d1..1d94bdfaff 100644 --- a/runtime/interpreter/storagemap.go +++ b/runtime/interpreter/storagemap.go @@ -125,12 +125,14 @@ func (s StorageMap) SetValue(interpreter *Interpreter, key StorageMapKey, value if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(s.orderedMap) + interpreter.maybeValidateAtreeStorage() existed = existingStorable != nil if existed { existingValue := StoredValue(interpreter, existingStorable, interpreter.Storage()) - existingValue.DeepRemove(interpreter) + existingValue.DeepRemove(interpreter, true) // existingValue is standalone because it was overwritten in parent container. interpreter.RemoveReferencedSlab(existingStorable) } return @@ -152,7 +154,9 @@ func (s StorageMap) RemoveValue(interpreter *Interpreter, key StorageMapKey) (ex } panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(s.orderedMap) + interpreter.maybeValidateAtreeStorage() // Key @@ -165,7 +169,7 @@ func (s StorageMap) RemoveValue(interpreter *Interpreter, key StorageMapKey) (ex existed = existingValueStorable != nil if existed { existingValue := StoredValue(interpreter, existingValueStorable, interpreter.Storage()) - existingValue.DeepRemove(interpreter) + existingValue.DeepRemove(interpreter, true) // existingValue is standalone because it was removed from parent container. interpreter.RemoveReferencedSlab(existingValueStorable) } return @@ -174,7 +178,10 @@ func (s StorageMap) RemoveValue(interpreter *Interpreter, key StorageMapKey) (ex // Iterator returns an iterator (StorageMapIterator), // which allows iterating over the keys and values of the storage map func (s StorageMap) Iterator(gauge common.MemoryGauge) StorageMapIterator { - mapIterator, err := s.orderedMap.Iterator() + mapIterator, err := s.orderedMap.Iterator( + StorageMapKeyAtreeValueComparator, + StorageMapKeyAtreeValueHashInput, + ) if err != nil { panic(errors.NewExternalError(err)) } diff --git a/runtime/interpreter/storagemapkey.go b/runtime/interpreter/storagemapkey.go index 37a6b783fd..7be319789f 100644 --- a/runtime/interpreter/storagemapkey.go +++ b/runtime/interpreter/storagemapkey.go @@ -18,7 +18,11 @@ package interpreter -import "github.com/onflow/atree" +import ( + "github.com/onflow/atree" + + "github.com/onflow/cadence/runtime/errors" +) type StorageMapKey interface { isStorageMapKey() @@ -72,3 +76,35 @@ func (Uint64StorageMapKey) AtreeValueCompare( func (k Uint64StorageMapKey) AtreeValue() atree.Value { return Uint64AtreeValue(k) } + +func StorageMapKeyAtreeValueHashInput(value atree.Value, scratch []byte) ([]byte, error) { + var smk StorageMapKey + switch value := value.(type) { + case StringAtreeValue: + smk = StringStorageMapKey(value) + + case Uint64AtreeValue: + smk = Uint64StorageMapKey(value) + + default: + return nil, errors.NewUnexpectedError("StorageMapKeyAtreeValueHashInput expected StringAtreeValue or Uint64AtreeValue, got %T", value) + } + + return smk.AtreeValueHashInput(value, scratch) +} + +func StorageMapKeyAtreeValueComparator(slabStorage atree.SlabStorage, value atree.Value, otherStorable atree.Storable) (bool, error) { + var smk StorageMapKey + switch value := value.(type) { + case StringAtreeValue: + smk = StringStorageMapKey(value) + + case Uint64AtreeValue: + smk = Uint64StorageMapKey(value) + + default: + return false, errors.NewUnexpectedError("StorageMapKeyAtreeValueComparator expected StringAtreeValue or Uint64AtreeValue, got %T", value) + } + + return smk.AtreeValueCompare(slabStorage, value, otherStorable) +} diff --git a/runtime/interpreter/stringatreevalue.go b/runtime/interpreter/stringatreevalue.go index 98e7fcf2f9..d0562608f9 100644 --- a/runtime/interpreter/stringatreevalue.go +++ b/runtime/interpreter/stringatreevalue.go @@ -28,6 +28,7 @@ type StringAtreeValue string var _ atree.Value = StringAtreeValue("") var _ atree.Storable = StringAtreeValue("") +var _ atree.ComparableStorable = StringAtreeValue("") func (v StringAtreeValue) Storable( storage atree.SlabStorage, @@ -57,6 +58,26 @@ func (StringAtreeValue) ChildStorables() []atree.Storable { return nil } +// Equal returns true if the given storable is equal to this StringAtreeValue. +func (v StringAtreeValue) Equal(other atree.Storable) bool { + v1, ok := other.(StringAtreeValue) + return ok && v == v1 +} + +// Less returns true if the given storable is less than StringAtreeValue. +func (v StringAtreeValue) Less(other atree.Storable) bool { + return v < other.(StringAtreeValue) +} + +// ID returns a unique identifier. +func (v StringAtreeValue) ID() string { + return string(v) +} + +func (v StringAtreeValue) Copy() atree.Storable { + return v +} + func StringAtreeValueHashInput(v atree.Value, _ []byte) ([]byte, error) { return []byte(v.(StringAtreeValue)), nil } diff --git a/runtime/interpreter/value.go b/runtime/interpreter/value.go index 238aa2da1e..7768bdccbd 100644 --- a/runtime/interpreter/value.go +++ b/runtime/interpreter/value.go @@ -107,8 +107,8 @@ type Value interface { // NOTE: important, error messages rely on values to implement String fmt.Stringer isValue() - Accept(interpreter *Interpreter, visitor Visitor) - Walk(interpreter *Interpreter, walkChild func(Value)) + Accept(interpreter *Interpreter, locationRange LocationRange, visitor Visitor) + Walk(interpreter *Interpreter, locationRange LocationRange, walkChild func(Value)) StaticType(interpreter *Interpreter) StaticType // ConformsToStaticType returns true if the value (i.e. its dynamic type) // conforms to its own static type. @@ -134,8 +134,12 @@ type Value interface { remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, // hasNoParentContainer is true when transferred value isn't an element of another container. ) Value - DeepRemove(interpreter *Interpreter) + DeepRemove( + interpreter *Interpreter, + hasNoParentContainer bool, // hasNoParentContainer is true when transferred value isn't an element of another container. + ) // Clone returns a new value that is equal to this value. // NOTE: not used by interpreter, but used externally (e.g. state migration) // NOTE: memory metering is unnecessary for Clone methods @@ -335,11 +339,11 @@ func NewTypeValue( func (TypeValue) isValue() {} -func (v TypeValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v TypeValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitTypeValue(interpreter, v) } -func (TypeValue) Walk(_ *Interpreter, _ func(Value)) { +func (TypeValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -484,6 +488,7 @@ func (v TypeValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -495,7 +500,7 @@ func (v TypeValue) Clone(_ *Interpreter) Value { return v } -func (TypeValue) DeepRemove(_ *Interpreter) { +func (TypeValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -543,11 +548,11 @@ var _ EquatableValue = VoidValue{} func (VoidValue) isValue() {} -func (v VoidValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v VoidValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitVoidValue(interpreter, v) } -func (VoidValue) Walk(_ *Interpreter, _ func(Value)) { +func (VoidValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -604,6 +609,7 @@ func (v VoidValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -615,7 +621,7 @@ func (v VoidValue) Clone(_ *Interpreter) Value { return v } -func (VoidValue) DeepRemove(_ *Interpreter) { +func (VoidValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -652,11 +658,11 @@ func AsBoolValue(v bool) BoolValue { func (BoolValue) isValue() {} -func (v BoolValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v BoolValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitBoolValue(interpreter, v) } -func (BoolValue) Walk(_ *Interpreter, _ func(Value)) { +func (BoolValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -777,6 +783,7 @@ func (v BoolValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -788,7 +795,7 @@ func (v BoolValue) Clone(_ *Interpreter) Value { return v } -func (BoolValue) DeepRemove(_ *Interpreter) { +func (BoolValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -835,11 +842,11 @@ var _ MemberAccessibleValue = CharacterValue("a") func (CharacterValue) isValue() {} -func (v CharacterValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v CharacterValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitCharacterValue(interpreter, v) } -func (CharacterValue) Walk(_ *Interpreter, _ func(Value)) { +func (CharacterValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -951,6 +958,7 @@ func (v CharacterValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -962,7 +970,7 @@ func (v CharacterValue) Clone(_ *Interpreter) Value { return v } -func (CharacterValue) DeepRemove(_ *Interpreter) { +func (CharacterValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -1066,11 +1074,11 @@ func (v *StringValue) prepareGraphemes() { func (*StringValue) isValue() {} -func (v *StringValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *StringValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitStringValue(interpreter, v) } -func (*StringValue) Walk(_ *Interpreter, _ func(Value)) { +func (*StringValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -1432,6 +1440,7 @@ func (v *StringValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -1443,7 +1452,7 @@ func (v *StringValue) Clone(_ *Interpreter) Value { return NewUnmeteredStringValue(v.Str) } -func (*StringValue) DeepRemove(_ *Interpreter) { +func (*StringValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -1609,6 +1618,7 @@ func NewArrayValue( true, nil, nil, + true, // standalone value doesn't have parent container. ) return value @@ -1710,15 +1720,19 @@ var _ IterableValue = &ArrayValue{} func (*ArrayValue) isValue() {} -func (v *ArrayValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *ArrayValue) Accept(interpreter *Interpreter, locationRange LocationRange, visitor Visitor) { descend := visitor.VisitArrayValue(interpreter, v) if !descend { return } - v.Walk(interpreter, func(element Value) { - element.Accept(interpreter, visitor) - }) + v.Walk( + interpreter, + locationRange, + func(element Value) { + element.Accept(interpreter, locationRange, visitor) + }, + ) } func (v *ArrayValue) Iterate(interpreter *Interpreter, f func(element Value) (resume bool)) { @@ -1743,11 +1757,13 @@ func (v *ArrayValue) Iterate(interpreter *Interpreter, f func(element Value) (re } } -func (v *ArrayValue) Walk(interpreter *Interpreter, walkChild func(Value)) { - v.Iterate(interpreter, func(element Value) (resume bool) { - walkChild(element) - return true - }) +func (v *ArrayValue) Walk(interpreter *Interpreter, _ LocationRange, walkChild func(Value)) { + v.Iterate( + interpreter, + func(element Value) (resume bool) { + walkChild(element) + return true + }) } func (v *ArrayValue) StaticType(_ *Interpreter) StaticType { @@ -1810,9 +1826,13 @@ func (v *ArrayValue) Destroy(interpreter *Interpreter, locationRange LocationRan valueID, locationRange, func() { - v.Walk(interpreter, func(element Value) { - maybeDestroy(interpreter, locationRange, element) - }) + v.Walk( + interpreter, + locationRange, + func(element Value) { + maybeDestroy(interpreter, locationRange, element) + }, + ) }, ) @@ -1906,6 +1926,7 @@ func (v *ArrayValue) Concat(interpreter *Interpreter, locationRange LocationRang false, nil, nil, + false, // value has a parent container because it is from iterator. ) }, ) @@ -1995,6 +2016,7 @@ func (v *ArrayValue) Set(interpreter *Interpreter, locationRange LocationRange, map[atree.ValueID]struct{}{ v.ValueID(): {}, }, + true, // standalone element doesn't have a parent container yet. ) existingStorable, err := v.array.Set(uint64(index), element) @@ -2003,11 +2025,13 @@ func (v *ArrayValue) Set(interpreter *Interpreter, locationRange LocationRange, panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + interpreter.maybeValidateAtreeStorage() existingValue := StoredValue(interpreter, existingStorable, interpreter.Storage()) - existingValue.DeepRemove(interpreter) + existingValue.DeepRemove(interpreter, true) // existingValue is standalone because it was overwritten in parent container. interpreter.RemoveReferencedSlab(existingStorable) } @@ -2033,7 +2057,7 @@ func (v *ArrayValue) MeteredString(memoryGauge common.MemoryGauge, seenReference i := 0 - _ = v.array.Iterate(func(element atree.Value) (resume bool, err error) { + _ = v.array.IterateReadOnly(func(element atree.Value) (resume bool, err error) { // ok to not meter anything created as part of this iteration, since we will discard the result // upon creating the string values[i] = MustConvertUnmeteredStoredValue(element).MeteredString(memoryGauge, seenReferences) @@ -2069,19 +2093,26 @@ func (v *ArrayValue) Append(interpreter *Interpreter, locationRange LocationRang map[atree.ValueID]struct{}{ v.ValueID(): {}, }, + true, // standalone element doesn't have a parent container yet. ) err := v.array.Append(element) if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + interpreter.maybeValidateAtreeStorage() } func (v *ArrayValue) AppendAll(interpreter *Interpreter, locationRange LocationRange, other *ArrayValue) { - other.Walk(interpreter, func(value Value) { - v.Append(interpreter, locationRange, value) - }) + other.Walk( + interpreter, + locationRange, + func(value Value) { + v.Append(interpreter, locationRange, value) + }, + ) } func (v *ArrayValue) InsertKey(interpreter *Interpreter, locationRange LocationRange, key Value, value Value) { @@ -2131,6 +2162,7 @@ func (v *ArrayValue) Insert(interpreter *Interpreter, locationRange LocationRang map[atree.ValueID]struct{}{ v.ValueID(): {}, }, + true, // standalone element doesn't have a parent container yet. ) err := v.array.Insert(uint64(index), element) @@ -2139,7 +2171,9 @@ func (v *ArrayValue) Insert(interpreter *Interpreter, locationRange LocationRang panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + interpreter.maybeValidateAtreeStorage() } func (v *ArrayValue) RemoveKey(interpreter *Interpreter, locationRange LocationRange, key Value) Value { @@ -2174,7 +2208,9 @@ func (v *ArrayValue) Remove(interpreter *Interpreter, locationRange LocationRang panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + interpreter.maybeValidateAtreeStorage() value := StoredValue(interpreter, storable, interpreter.Storage()) @@ -2185,6 +2221,7 @@ func (v *ArrayValue) Remove(interpreter *Interpreter, locationRange LocationRang true, storable, nil, + true, // value is standalone because it was removed from parent container. ) } @@ -2654,6 +2691,7 @@ func (v *ArrayValue) Transfer( remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { baseUsage, elementUsage, dataSlabs, metaDataSlabs := common.NewArrayMemoryUsages(v.array.Count(), v.elementSize) common.UseMemory(interpreter, baseUsage) @@ -2703,6 +2741,8 @@ func (v *ArrayValue) Transfer( if needsStoreTo || !isResourceKinded { + // Use non-readonly iterator here because iterated + // value can be removed if remove parameter is true. iterator, err := v.array.Iterator() if err != nil { panic(errors.NewExternalError(err)) @@ -2722,7 +2762,15 @@ func (v *ArrayValue) Transfer( } element := MustConvertStoredValue(interpreter, value). - Transfer(interpreter, locationRange, address, remove, nil, preventTransfer) + Transfer( + interpreter, + locationRange, + address, + remove, + nil, + preventTransfer, + false, // value has a parent container because it is from iterator. + ) return element, nil }, @@ -2738,7 +2786,11 @@ func (v *ArrayValue) Transfer( if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } interpreter.RemoveReferencedSlab(storable) } @@ -2792,7 +2844,7 @@ func (v *ArrayValue) Transfer( func (v *ArrayValue) Clone(interpreter *Interpreter) Value { config := interpreter.SharedState.Config - iterator, err := v.array.Iterator() + iterator, err := v.array.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -2834,7 +2886,7 @@ func (v *ArrayValue) Clone(interpreter *Interpreter) Value { } } -func (v *ArrayValue) DeepRemove(interpreter *Interpreter) { +func (v *ArrayValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { config := interpreter.SharedState.Config if config.TracingEnabled { @@ -2858,13 +2910,17 @@ func (v *ArrayValue) DeepRemove(interpreter *Interpreter) { err := v.array.PopIterate(func(storable atree.Storable) { value := StoredValue(interpreter, storable, storage) - value.DeepRemove(interpreter) + value.DeepRemove(interpreter, false) // existingValue is an element of v.array because it is from PopIterate() callback. interpreter.RemoveReferencedSlab(storable) }) if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.array) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } } func (v *ArrayValue) SlabID() atree.SlabID { @@ -2978,6 +3034,7 @@ func (v *ArrayValue) Slice( false, nil, nil, + false, // value has a parent container because it is from iterator. ) }, ) @@ -3013,6 +3070,7 @@ func (v *ArrayValue) Reverse( false, nil, nil, + false, // value has a parent container because it is returned by Get(). ) }, ) @@ -3089,6 +3147,7 @@ func (v *ArrayValue) Filter( false, nil, nil, + false, // value has a parent container because it is from iterator. ) }, ) @@ -3171,6 +3230,7 @@ func (v *ArrayValue) Map( false, nil, nil, + false, // value has a parent container because it is from iterator. ) }, ) @@ -3388,11 +3448,11 @@ var _ MemberAccessibleValue = IntValue{} func (IntValue) isValue() {} -func (v IntValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v IntValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitIntValue(interpreter, v) } -func (IntValue) Walk(_ *Interpreter, _ func(Value)) { +func (IntValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -3894,6 +3954,7 @@ func (v IntValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -3905,7 +3966,7 @@ func (v IntValue) Clone(_ *Interpreter) Value { return NewUnmeteredIntValueFromBigInt(v.BigInt) } -func (IntValue) DeepRemove(_ *Interpreter) { +func (IntValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -3949,11 +4010,11 @@ var _ HashableValue = Int8Value(0) func (Int8Value) isValue() {} -func (v Int8Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int8Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt8Value(interpreter, v) } -func (Int8Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int8Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -4534,6 +4595,7 @@ func (v Int8Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -4545,7 +4607,7 @@ func (v Int8Value) Clone(_ *Interpreter) Value { return v } -func (Int8Value) DeepRemove(_ *Interpreter) { +func (Int8Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -4590,11 +4652,11 @@ var _ MemberAccessibleValue = Int16Value(0) func (Int16Value) isValue() {} -func (v Int16Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int16Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt16Value(interpreter, v) } -func (Int16Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int16Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -5176,6 +5238,7 @@ func (v Int16Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -5187,7 +5250,7 @@ func (v Int16Value) Clone(_ *Interpreter) Value { return v } -func (Int16Value) DeepRemove(_ *Interpreter) { +func (Int16Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -5232,11 +5295,11 @@ var _ MemberAccessibleValue = Int32Value(0) func (Int32Value) isValue() {} -func (v Int32Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int32Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt32Value(interpreter, v) } -func (Int32Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int32Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -5818,6 +5881,7 @@ func (v Int32Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -5829,7 +5893,7 @@ func (v Int32Value) Clone(_ *Interpreter) Value { return v } -func (Int32Value) DeepRemove(_ *Interpreter) { +func (Int32Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -5872,11 +5936,11 @@ var _ MemberAccessibleValue = Int64Value(0) func (Int64Value) isValue() {} -func (v Int64Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int64Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt64Value(interpreter, v) } -func (Int64Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int64Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -6452,6 +6516,7 @@ func (v Int64Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -6463,7 +6528,7 @@ func (v Int64Value) Clone(_ *Interpreter) Value { return v } -func (Int64Value) DeepRemove(_ *Interpreter) { +func (Int64Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -6523,11 +6588,11 @@ var _ MemberAccessibleValue = Int128Value{} func (Int128Value) isValue() {} -func (v Int128Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int128Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt128Value(interpreter, v) } -func (Int128Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int128Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -7196,6 +7261,7 @@ func (v Int128Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -7207,7 +7273,7 @@ func (v Int128Value) Clone(_ *Interpreter) Value { return NewUnmeteredInt128ValueFromBigInt(v.BigInt) } -func (Int128Value) DeepRemove(_ *Interpreter) { +func (Int128Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -7267,11 +7333,11 @@ var _ MemberAccessibleValue = Int256Value{} func (Int256Value) isValue() {} -func (v Int256Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Int256Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInt256Value(interpreter, v) } -func (Int256Value) Walk(_ *Interpreter, _ func(Value)) { +func (Int256Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -7937,6 +8003,7 @@ func (v Int256Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -7948,7 +8015,7 @@ func (v Int256Value) Clone(_ *Interpreter) Value { return NewUnmeteredInt256ValueFromBigInt(v.BigInt) } -func (Int256Value) DeepRemove(_ *Interpreter) { +func (Int256Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -8049,11 +8116,11 @@ var _ MemberAccessibleValue = UIntValue{} func (UIntValue) isValue() {} -func (v UIntValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UIntValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUIntValue(interpreter, v) } -func (UIntValue) Walk(_ *Interpreter, _ func(Value)) { +func (UIntValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -8566,6 +8633,7 @@ func (v UIntValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -8577,7 +8645,7 @@ func (v UIntValue) Clone(_ *Interpreter) Value { return NewUnmeteredUIntValueFromBigInt(v.BigInt) } -func (UIntValue) DeepRemove(_ *Interpreter) { +func (UIntValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -8620,11 +8688,11 @@ func NewUnmeteredUInt8Value(value uint8) UInt8Value { func (UInt8Value) isValue() {} -func (v UInt8Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt8Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt8Value(interpreter, v) } -func (UInt8Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt8Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -9152,6 +9220,7 @@ func (v UInt8Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -9163,7 +9232,7 @@ func (v UInt8Value) Clone(_ *Interpreter) Value { return v } -func (UInt8Value) DeepRemove(_ *Interpreter) { +func (UInt8Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -9206,11 +9275,11 @@ func NewUnmeteredUInt16Value(value uint16) UInt16Value { func (UInt16Value) isValue() {} -func (v UInt16Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt16Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt16Value(interpreter, v) } -func (UInt16Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt16Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -9693,6 +9762,7 @@ func (v UInt16Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -9704,7 +9774,7 @@ func (v UInt16Value) Clone(_ *Interpreter) Value { return v } -func (UInt16Value) DeepRemove(_ *Interpreter) { +func (UInt16Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -9747,11 +9817,11 @@ var _ MemberAccessibleValue = UInt32Value(0) func (UInt32Value) isValue() {} -func (v UInt32Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt32Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt32Value(interpreter, v) } -func (UInt32Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt32Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -10235,6 +10305,7 @@ func (v UInt32Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -10246,7 +10317,7 @@ func (v UInt32Value) Clone(_ *Interpreter) Value { return v } -func (UInt32Value) DeepRemove(_ *Interpreter) { +func (UInt32Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -10295,11 +10366,11 @@ func NewUnmeteredUInt64Value(value uint64) UInt64Value { func (UInt64Value) isValue() {} -func (v UInt64Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt64Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt64Value(interpreter, v) } -func (UInt64Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt64Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -10806,6 +10877,7 @@ func (v UInt64Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -10817,7 +10889,7 @@ func (v UInt64Value) Clone(_ *Interpreter) Value { return v } -func (UInt64Value) DeepRemove(_ *Interpreter) { +func (UInt64Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -10877,11 +10949,11 @@ var _ MemberAccessibleValue = UInt128Value{} func (UInt128Value) isValue() {} -func (v UInt128Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt128Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt128Value(interpreter, v) } -func (UInt128Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt128Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -11481,6 +11553,7 @@ func (v UInt128Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -11492,7 +11565,7 @@ func (v UInt128Value) Clone(_ *Interpreter) Value { return NewUnmeteredUInt128ValueFromBigInt(v.BigInt) } -func (UInt128Value) DeepRemove(_ *Interpreter) { +func (UInt128Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -11552,11 +11625,11 @@ var _ MemberAccessibleValue = UInt256Value{} func (UInt256Value) isValue() {} -func (v UInt256Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UInt256Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUInt256Value(interpreter, v) } -func (UInt256Value) Walk(_ *Interpreter, _ func(Value)) { +func (UInt256Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -12155,6 +12228,7 @@ func (v UInt256Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -12166,7 +12240,7 @@ func (v UInt256Value) Clone(_ *Interpreter) Value { return NewUnmeteredUInt256ValueFromBigInt(v.BigInt) } -func (UInt256Value) DeepRemove(_ *Interpreter) { +func (UInt256Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -12211,11 +12285,11 @@ func NewUnmeteredWord8Value(value uint8) Word8Value { func (Word8Value) isValue() {} -func (v Word8Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word8Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord8Value(interpreter, v) } -func (Word8Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word8Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -12592,6 +12666,7 @@ func (v Word8Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -12603,7 +12678,7 @@ func (v Word8Value) Clone(_ *Interpreter) Value { return v } -func (Word8Value) DeepRemove(_ *Interpreter) { +func (Word8Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -12648,11 +12723,11 @@ func NewUnmeteredWord16Value(value uint16) Word16Value { func (Word16Value) isValue() {} -func (v Word16Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word16Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord16Value(interpreter, v) } -func (Word16Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word16Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -13030,6 +13105,7 @@ func (v Word16Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -13041,7 +13117,7 @@ func (v Word16Value) Clone(_ *Interpreter) Value { return v } -func (Word16Value) DeepRemove(_ *Interpreter) { +func (Word16Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -13086,11 +13162,11 @@ func NewUnmeteredWord32Value(value uint32) Word32Value { func (Word32Value) isValue() {} -func (v Word32Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word32Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord32Value(interpreter, v) } -func (Word32Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word32Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -13469,6 +13545,7 @@ func (v Word32Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -13480,7 +13557,7 @@ func (v Word32Value) Clone(_ *Interpreter) Value { return v } -func (Word32Value) DeepRemove(_ *Interpreter) { +func (Word32Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -13531,11 +13608,11 @@ var _ BigNumberValue = Word64Value(0) func (Word64Value) isValue() {} -func (v Word64Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word64Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord64Value(interpreter, v) } -func (Word64Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word64Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -13934,6 +14011,7 @@ func (v Word64Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -13949,7 +14027,7 @@ func (v Word64Value) ByteSize() uint32 { return cborTagSize + getUintCBORSize(uint64(v)) } -func (Word64Value) DeepRemove(_ *Interpreter) { +func (Word64Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -14005,11 +14083,11 @@ var _ MemberAccessibleValue = Word128Value{} func (Word128Value) isValue() {} -func (v Word128Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word128Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord128Value(interpreter, v) } -func (Word128Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word128Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -14514,6 +14592,7 @@ func (v Word128Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -14525,7 +14604,7 @@ func (v Word128Value) Clone(_ *Interpreter) Value { return NewUnmeteredWord128ValueFromBigInt(v.BigInt) } -func (Word128Value) DeepRemove(_ *Interpreter) { +func (Word128Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -14585,11 +14664,11 @@ var _ MemberAccessibleValue = Word256Value{} func (Word256Value) isValue() {} -func (v Word256Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Word256Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitWord256Value(interpreter, v) } -func (Word256Value) Walk(_ *Interpreter, _ func(Value)) { +func (Word256Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -15095,6 +15174,7 @@ func (v Word256Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -15106,7 +15186,7 @@ func (v Word256Value) Clone(_ *Interpreter) Value { return NewUnmeteredWord256ValueFromBigInt(v.BigInt) } -func (Word256Value) DeepRemove(_ *Interpreter) { +func (Word256Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -15180,11 +15260,11 @@ var _ MemberAccessibleValue = Fix64Value(0) func (Fix64Value) isValue() {} -func (v Fix64Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v Fix64Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitFix64Value(interpreter, v) } -func (Fix64Value) Walk(_ *Interpreter, _ func(Value)) { +func (Fix64Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -15672,6 +15752,7 @@ func (v Fix64Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -15683,7 +15764,7 @@ func (v Fix64Value) Clone(_ *Interpreter) Value { return v } -func (Fix64Value) DeepRemove(_ *Interpreter) { +func (Fix64Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -15751,11 +15832,11 @@ var _ MemberAccessibleValue = UFix64Value(0) func (UFix64Value) isValue() {} -func (v UFix64Value) Accept(interpreter *Interpreter, visitor Visitor) { +func (v UFix64Value) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitUFix64Value(interpreter, v) } -func (UFix64Value) Walk(_ *Interpreter, _ func(Value)) { +func (UFix64Value) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -16206,6 +16287,7 @@ func (v UFix64Value) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -16217,7 +16299,7 @@ func (v UFix64Value) Clone(_ *Interpreter) Value { return v } -func (UFix64Value) DeepRemove(_ *Interpreter) { +func (UFix64Value) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -16402,14 +16484,14 @@ var _ ContractValue = &CompositeValue{} func (*CompositeValue) isValue() {} -func (v *CompositeValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *CompositeValue) Accept(interpreter *Interpreter, locationRange LocationRange, visitor Visitor) { descend := visitor.VisitCompositeValue(interpreter, v) if !descend { return } v.ForEachField(interpreter, func(_ string, value Value) (resume bool) { - value.Accept(interpreter, visitor) + value.Accept(interpreter, locationRange, visitor) // continue iteration return true @@ -16418,7 +16500,7 @@ func (v *CompositeValue) Accept(interpreter *Interpreter, visitor Visitor) { // Walk iterates over all field values of the composite value. // It does NOT walk the computed field or functions! -func (v *CompositeValue) Walk(interpreter *Interpreter, walkChild func(Value)) { +func (v *CompositeValue) Walk(interpreter *Interpreter, _ LocationRange, walkChild func(Value)) { v.ForEachField(interpreter, func(_ string, value Value) (resume bool) { walkChild(value) @@ -16767,7 +16849,9 @@ func (v *CompositeValue) RemoveMember( } panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + interpreter.maybeValidateAtreeStorage() // Key interpreter.RemoveReferencedSlab(existingKeyStorable) @@ -16787,6 +16871,7 @@ func (v *CompositeValue) RemoveMember( true, existingValueStorable, nil, + true, // value is standalone because it was removed from parent container. ) } @@ -16835,6 +16920,7 @@ func (v *CompositeValue) SetMember( map[atree.ValueID]struct{}{ valueID: {}, }, + true, // value is standalone before being set in parent container. ) existingStorable, err := v.dictionary.Set( @@ -16846,12 +16932,14 @@ func (v *CompositeValue) SetMember( if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + interpreter.maybeValidateAtreeStorage() if existingStorable != nil { existingValue := StoredValue(interpreter, existingStorable, config.Storage) - existingValue.DeepRemove(interpreter) + existingValue.DeepRemove(interpreter, true) // existingValue is standalone because it was overwritten in parent container. interpreter.RemoveReferencedSlab(existingStorable) return true @@ -16879,7 +16967,7 @@ func (v *CompositeValue) MeteredString(memoryGauge common.MemoryGauge, seenRefer strLen := emptyCompositeStringLen var fields []CompositeField - _ = v.dictionary.Iterate(func(key atree.Value, value atree.Value) (resume bool, err error) { + _ = v.dictionary.IterateReadOnly(func(key atree.Value, value atree.Value) (resume bool, err error) { field := NewCompositeField( memoryGauge, string(key.(StringAtreeValue)), @@ -16971,7 +17059,7 @@ func (v *CompositeValue) Equal(interpreter *Interpreter, locationRange LocationR return false } - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -17185,6 +17273,7 @@ func (v *CompositeValue) Transfer( remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { config := interpreter.SharedState.Config @@ -17244,7 +17333,12 @@ func (v *CompositeValue) Transfer( } if needsStoreTo || !isResourceKinded { - iterator, err := v.dictionary.Iterator() + // Use non-readonly iterator here because iterated + // value can be removed if remove parameter is true. + iterator, err := v.dictionary.Iterator( + StringAtreeValueComparator, + StringAtreeValueHashInput, + ) if err != nil { panic(errors.NewExternalError(err)) } @@ -17289,6 +17383,7 @@ func (v *CompositeValue) Transfer( remove, nil, preventTransfer, + false, // value is an element of parent container because it is returned from iterator. ) return atreeKey, value, nil @@ -17306,7 +17401,11 @@ func (v *CompositeValue) Transfer( if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } interpreter.RemoveReferencedSlab(storable) } @@ -17394,7 +17493,7 @@ func (v *CompositeValue) ResourceUUID(interpreter *Interpreter, locationRange Lo func (v *CompositeValue) Clone(interpreter *Interpreter) Value { - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -17453,7 +17552,7 @@ func (v *CompositeValue) Clone(interpreter *Interpreter) Value { } } -func (v *CompositeValue) DeepRemove(interpreter *Interpreter) { +func (v *CompositeValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { config := interpreter.SharedState.Config if config.TracingEnabled { @@ -17483,13 +17582,17 @@ func (v *CompositeValue) DeepRemove(interpreter *Interpreter) { interpreter.RemoveReferencedSlab(nameStorable) value := StoredValue(interpreter, valueStorable, storage) - value.DeepRemove(interpreter) + value.DeepRemove(interpreter, false) // value is an element of v.dictionary because it is from PopIterate() callback. interpreter.RemoveReferencedSlab(valueStorable) }) if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } } func (v *CompositeValue) GetOwner() common.Address { @@ -17503,13 +17606,16 @@ func (v *CompositeValue) ForEachField( f func(fieldName string, fieldValue Value) (resume bool), ) { - err := v.dictionary.Iterate(func(key atree.Value, value atree.Value) (resume bool, err error) { - resume = f( - string(key.(StringAtreeValue)), - MustConvertStoredValue(gauge, value), - ) - return - }) + err := v.dictionary.Iterate( + StringAtreeValueComparator, + StringAtreeValueHashInput, + func(key atree.Value, value atree.Value) (resume bool, err error) { + resume = f( + string(key.(StringAtreeValue)), + MustConvertStoredValue(gauge, value), + ) + return + }) if err != nil { panic(errors.NewExternalError(err)) } @@ -17545,7 +17651,9 @@ func (v *CompositeValue) RemoveField( } panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + interpreter.maybeValidateAtreeStorage() // Key @@ -17555,7 +17663,7 @@ func (v *CompositeValue) RemoveField( // Value existingValue := StoredValue(interpreter, existingValueStorable, interpreter.Storage()) - existingValue.DeepRemove(interpreter) + existingValue.DeepRemove(interpreter, true) // existingValue is standalone because it was removed from parent container. interpreter.RemoveReferencedSlab(existingValueStorable) } @@ -17648,7 +17756,10 @@ func attachmentBaseAndSelfValues( } func (v *CompositeValue) forEachAttachment(interpreter *Interpreter, _ LocationRange, f func(*CompositeValue)) { - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.Iterator( + StringAtreeValueComparator, + StringAtreeValueHashInput, + ) if err != nil { panic(errors.NewExternalError(err)) } @@ -17874,30 +17985,49 @@ var _ ReferenceTrackedResourceKindedValue = &DictionaryValue{} func (*DictionaryValue) isValue() {} -func (v *DictionaryValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *DictionaryValue) Accept( + interpreter *Interpreter, + locationRange LocationRange, + visitor Visitor, +) { descend := visitor.VisitDictionaryValue(interpreter, v) if !descend { return } - v.Walk(interpreter, func(value Value) { - value.Accept(interpreter, visitor) - }) + v.Walk( + interpreter, + locationRange, + func(value Value) { + value.Accept(interpreter, locationRange, visitor) + }, + ) } -func (v *DictionaryValue) Iterate(interpreter *Interpreter, f func(key, value Value) (resume bool)) { +func (v *DictionaryValue) Iterate( + interpreter *Interpreter, + locationRange LocationRange, + f func(key, value Value) (resume bool), +) { + + valueComparator := newValueComparator(interpreter, locationRange) + hashInputProvider := newHashInputProvider(interpreter, locationRange) + iterate := func() { - err := v.dictionary.Iterate(func(key, value atree.Value) (resume bool, err error) { - // atree.OrderedMap iteration provides low-level atree.Value, - // convert to high-level interpreter.Value + err := v.dictionary.Iterate( + valueComparator, + hashInputProvider, + func(key, value atree.Value) (resume bool, err error) { + // atree.OrderedMap iteration provides low-level atree.Value, + // convert to high-level interpreter.Value - resume = f( - MustConvertStoredValue(interpreter, key), - MustConvertStoredValue(interpreter, value), - ) + resume = f( + MustConvertStoredValue(interpreter, key), + MustConvertStoredValue(interpreter, value), + ) - return resume, nil - }) + return resume, nil + }) if err != nil { panic(errors.NewExternalError(err)) } @@ -17909,11 +18039,11 @@ func (v *DictionaryValue) Iterate(interpreter *Interpreter, f func(key, value Va } } -type DictionaryIterator struct { +type DictionaryKeyIterator struct { mapIterator *atree.MapIterator } -func (i DictionaryIterator) NextKey(gauge common.MemoryGauge) Value { +func (i DictionaryKeyIterator) NextKey(gauge common.MemoryGauge) Value { atreeValue, err := i.mapIterator.NextKey() if err != nil { panic(errors.NewExternalError(err)) @@ -17924,23 +18054,32 @@ func (i DictionaryIterator) NextKey(gauge common.MemoryGauge) Value { return MustConvertStoredValue(gauge, atreeValue) } -func (v *DictionaryValue) Iterator() DictionaryIterator { - mapIterator, err := v.dictionary.Iterator() +func (v *DictionaryValue) Iterator() DictionaryKeyIterator { + + mapIterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } - return DictionaryIterator{ + return DictionaryKeyIterator{ mapIterator: mapIterator, } } -func (v *DictionaryValue) Walk(interpreter *Interpreter, walkChild func(Value)) { - v.Iterate(interpreter, func(key, value Value) (resume bool) { - walkChild(key) - walkChild(value) - return true - }) +func (v *DictionaryValue) Walk( + interpreter *Interpreter, + locationRange LocationRange, + walkChild func(Value), +) { + v.Iterate( + interpreter, + locationRange, + func(key, value Value) (resume bool) { + walkChild(key) + walkChild(value) + return true + }, + ) } func (v *DictionaryValue) StaticType(_ *Interpreter) StaticType { @@ -17950,16 +18089,19 @@ func (v *DictionaryValue) StaticType(_ *Interpreter) StaticType { func (v *DictionaryValue) IsImportable(inter *Interpreter) bool { importable := true - v.Iterate(inter, func(key, value Value) (resume bool) { - if !key.IsImportable(inter) || !value.IsImportable(inter) { - importable = false - // stop iteration - return false - } + v.Iterate( + inter, + EmptyLocationRange, + func(key, value Value) (resume bool) { + if !key.IsImportable(inter) || !value.IsImportable(inter) { + importable = false + // stop iteration + return false + } - // continue iteration - return true - }) + // continue iteration + return true + }) return importable } @@ -18007,13 +18149,17 @@ func (v *DictionaryValue) Destroy(interpreter *Interpreter, locationRange Locati valueID, locationRange, func() { - v.Iterate(interpreter, func(key, value Value) (resume bool) { - // Resources cannot be keys at the moment, so should theoretically not be needed - maybeDestroy(interpreter, locationRange, key) - maybeDestroy(interpreter, locationRange, value) + v.Iterate( + interpreter, + locationRange, + func(key, value Value) (resume bool) { + // Resources cannot be keys at the moment, so should theoretically not be needed + maybeDestroy(interpreter, locationRange, key) + maybeDestroy(interpreter, locationRange, value) - return true - }) + return true + }, + ) }, ) @@ -18061,7 +18207,7 @@ func (v *DictionaryValue) ForEachKey( } iterate := func() { - err := v.dictionary.IterateKeys( + err := v.dictionary.IterateReadOnlyKeys( func(item atree.Value) (bool, error) { key := MustConvertStoredValue(interpreter, item) @@ -18201,7 +18347,7 @@ func (v *DictionaryValue) MeteredString(memoryGauge common.MemoryGauge, seenRefe }, v.Count()) index := 0 - _ = v.dictionary.Iterate(func(key, value atree.Value) (resume bool, err error) { + _ = v.dictionary.IterateReadOnly(func(key, value atree.Value) (resume bool, err error) { // atree.OrderedMap iteration provides low-level atree.Value, // convert to high-level interpreter.Value @@ -18264,7 +18410,7 @@ func (v *DictionaryValue) GetMember( case "keys": - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -18292,13 +18438,17 @@ func (v *DictionaryValue) GetMember( false, nil, nil, + false, // value is an element of parent container because it is returned from iterator. ) }, ) case "values": - iterator, err := v.dictionary.Iterator() + valueComparator := newValueComparator(interpreter, locationRange) + hashInputProvider := newHashInputProvider(interpreter, locationRange) + + iterator, err := v.dictionary.Iterator(valueComparator, hashInputProvider) if err != nil { panic(errors.NewExternalError(err)) } @@ -18326,6 +18476,7 @@ func (v *DictionaryValue) GetMember( false, nil, nil, + false, // value is an element of parent container because it is returned from iterator. ) }) @@ -18472,14 +18623,16 @@ func (v *DictionaryValue) Remove( } panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + interpreter.maybeValidateAtreeStorage() storage := interpreter.Storage() // Key existingKeyValue := StoredValue(interpreter, existingKeyStorable, storage) - existingKeyValue.DeepRemove(interpreter) + existingKeyValue.DeepRemove(interpreter, true) // existingValue is standalone because it was removed from parent container. interpreter.RemoveReferencedSlab(existingKeyStorable) // Value @@ -18492,6 +18645,7 @@ func (v *DictionaryValue) Remove( true, existingValueStorable, nil, + true, // value is standalone because it was removed from parent container. ) return NewSomeValueNonCopying(interpreter, existingValue) @@ -18535,6 +18689,7 @@ func (v *DictionaryValue) Insert( true, nil, preventTransfer, + true, // keyValue is standalone before it is inserted into parent container. ) value = value.Transfer( @@ -18544,6 +18699,7 @@ func (v *DictionaryValue) Insert( true, nil, preventTransfer, + true, // value is standalone before it is inserted into parent container. ) valueComparator := newValueComparator(interpreter, locationRange) @@ -18560,7 +18716,9 @@ func (v *DictionaryValue) Insert( if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + interpreter.maybeValidateAtreeStorage() if existingValueStorable == nil { return NilOptionalValue @@ -18579,6 +18737,7 @@ func (v *DictionaryValue) Insert( true, existingValueStorable, nil, + true, // existingValueStorable is standalone after it is overwritten in parent container. ) return NewSomeValueNonCopying(interpreter, existingValue) @@ -18621,7 +18780,7 @@ func (v *DictionaryValue) ConformsToStaticType( keyType := staticType.KeyType valueType := staticType.ValueType - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -18688,7 +18847,7 @@ func (v *DictionaryValue) Equal(interpreter *Interpreter, locationRange Location return false } - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -18739,6 +18898,7 @@ func (v *DictionaryValue) Transfer( remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { baseUse, elementOverhead, dataUse, metaDataUse := common.NewDictionaryMemoryUsages( v.dictionary.Count(), @@ -18794,7 +18954,9 @@ func (v *DictionaryValue) Transfer( valueComparator := newValueComparator(interpreter, locationRange) hashInputProvider := newHashInputProvider(interpreter, locationRange) - iterator, err := v.dictionary.Iterator() + // Use non-readonly iterator here because iterated + // value can be removed if remove parameter is true. + iterator, err := v.dictionary.Iterator(valueComparator, hashInputProvider) if err != nil { panic(errors.NewExternalError(err)) } @@ -18821,10 +18983,26 @@ func (v *DictionaryValue) Transfer( } key := MustConvertStoredValue(interpreter, atreeKey). - Transfer(interpreter, locationRange, address, remove, nil, preventTransfer) + Transfer( + interpreter, + locationRange, + address, + remove, + nil, + preventTransfer, + false, // atreeKey has parent container because it is returned from iterator. + ) value := MustConvertStoredValue(interpreter, atreeValue). - Transfer(interpreter, locationRange, address, remove, nil, preventTransfer) + Transfer( + interpreter, + locationRange, + address, + remove, + nil, + preventTransfer, + false, // atreeValue has parent container because it is returned from iterator. + ) return key, value, nil }, @@ -18841,7 +19019,11 @@ func (v *DictionaryValue) Transfer( if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } interpreter.RemoveReferencedSlab(storable) } @@ -18898,7 +19080,7 @@ func (v *DictionaryValue) Clone(interpreter *Interpreter) Value { valueComparator := newValueComparator(interpreter, EmptyLocationRange) hashInputProvider := newHashInputProvider(interpreter, EmptyLocationRange) - iterator, err := v.dictionary.Iterator() + iterator, err := v.dictionary.ReadOnlyIterator() if err != nil { panic(errors.NewExternalError(err)) } @@ -18946,7 +19128,7 @@ func (v *DictionaryValue) Clone(interpreter *Interpreter) Value { } } -func (v *DictionaryValue) DeepRemove(interpreter *Interpreter) { +func (v *DictionaryValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { config := interpreter.SharedState.Config @@ -18972,17 +19154,21 @@ func (v *DictionaryValue) DeepRemove(interpreter *Interpreter) { err := v.dictionary.PopIterate(func(keyStorable atree.Storable, valueStorable atree.Storable) { key := StoredValue(interpreter, keyStorable, storage) - key.DeepRemove(interpreter) + key.DeepRemove(interpreter, false) // key is an element of v.dictionary because it is from PopIterate() callback. interpreter.RemoveReferencedSlab(keyStorable) value := StoredValue(interpreter, valueStorable, storage) - value.DeepRemove(interpreter) + value.DeepRemove(interpreter, false) // value is an element of v.dictionary because it is from PopIterate() callback. interpreter.RemoveReferencedSlab(valueStorable) }) if err != nil { panic(errors.NewExternalError(err)) } + interpreter.maybeValidateAtreeValue(v.dictionary) + if hasNoParentContainer { + interpreter.maybeValidateAtreeStorage() + } } func (v *DictionaryValue) GetOwner() common.Address { @@ -19046,11 +19232,11 @@ var _ OptionalValue = NilValue{} func (NilValue) isValue() {} -func (v NilValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v NilValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitNilValue(interpreter, v) } -func (NilValue) Walk(_ *Interpreter, _ func(Value)) { +func (NilValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -19162,6 +19348,7 @@ func (v NilValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -19173,7 +19360,7 @@ func (v NilValue) Clone(_ *Interpreter) Value { return v } -func (NilValue) DeepRemove(_ *Interpreter) { +func (NilValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -19217,15 +19404,15 @@ var _ OptionalValue = &SomeValue{} func (*SomeValue) isValue() {} -func (v *SomeValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *SomeValue) Accept(interpreter *Interpreter, locationRange LocationRange, visitor Visitor) { descend := visitor.VisitSomeValue(interpreter, v) if !descend { return } - v.value.Accept(interpreter, visitor) + v.value.Accept(interpreter, locationRange, visitor) } -func (v *SomeValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *SomeValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.value) } @@ -19403,9 +19590,26 @@ func (v *SomeValue) Storable( maxInlineSize uint64, ) (atree.Storable, error) { + // SomeStorable returned from this function can be encoded in two ways: + // - if non-SomeStorable is too large, non-SomeStorable is encoded in a separate slab + // while SomeStorable wrapper is encoded inline with reference to slab containing + // non-SomeStorable. + // - otherwise, SomeStorable with non-SomeStorable is encoded inline. + // + // The above applies to both immutable non-SomeValue (such as StringValue), + // and mutable non-SomeValue (such as ArrayValue). + if v.valueStorable == nil { - var err error - v.valueStorable, err = v.value.Storable( + + nonSomeValue, nestedLevels := v.nonSomeValue() + + someStorableEncodedPrefixSize := getSomeStorableEncodedPrefixSize(nestedLevels) + + // Reduce maxInlineSize for non-SomeValue to make sure + // that SomeStorable wrapper is always encoded inline. + maxInlineSize -= uint64(someStorableEncodedPrefixSize) + + nonSomeValueStorable, err := nonSomeValue.Storable( storage, address, maxInlineSize, @@ -19413,16 +19617,44 @@ func (v *SomeValue) Storable( if err != nil { return nil, err } + + valueStorable := nonSomeValueStorable + for i := 1; i < int(nestedLevels); i++ { + valueStorable = SomeStorable{ + Storable: valueStorable, + } + } + v.valueStorable = valueStorable } - return maybeLargeImmutableStorable( - SomeStorable{ - Storable: v.valueStorable, - }, - storage, - address, - maxInlineSize, - ) + // No need to call maybeLargeImmutableStorable() here for SomeStorable because: + // - encoded SomeStorable size = someStorableEncodedPrefixSize + non-SomeValueStorable size + // - non-SomeValueStorable size < maxInlineSize - someStorableEncodedPrefixSize + return SomeStorable{ + Storable: v.valueStorable, + }, nil +} + +// nonSomeValue returns a non-SomeValue and nested levels of SomeValue reached +// by traversing nested SomeValue (SomeValue containing SomeValue, etc.) +// until it reaches a non-SomeValue. +// For example, +// - `SomeValue{true}` has non-SomeValue `true`, and nested levels 1 +// - `SomeValue{SomeValue{1}}` has non-SomeValue `1` and nested levels 2 +// - `SomeValue{SomeValue{[SomeValue{SomeValue{SomeValue{1}}}]}} has +// non-SomeValue `[SomeValue{SomeValue{SomeValue{1}}}]` and nested levels 2 +func (v *SomeValue) nonSomeValue() (atree.Value, uint64) { + nestedLevels := uint64(1) + for { + switch value := v.value.(type) { + case *SomeValue: + nestedLevels++ + v = value + + default: + return value, nestedLevels + } + } } func (v *SomeValue) NeedsStoreTo(address atree.Address) bool { @@ -19453,6 +19685,7 @@ func (v *SomeValue) Transfer( remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { config := interpreter.SharedState.Config @@ -19474,6 +19707,7 @@ func (v *SomeValue) Transfer( remove, nil, preventTransfer, + hasNoParentContainer, ) if remove { @@ -19518,8 +19752,8 @@ func (v *SomeValue) Clone(interpreter *Interpreter) Value { return NewUnmeteredSomeValueNonCopying(innerValue) } -func (v *SomeValue) DeepRemove(interpreter *Interpreter) { - v.value.DeepRemove(interpreter) +func (v *SomeValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { + v.value.DeepRemove(interpreter, hasNoParentContainer) if v.valueStorable != nil { interpreter.RemoveReferencedSlab(v.valueStorable) } @@ -19540,10 +19774,49 @@ type SomeStorable struct { Storable atree.Storable } -var _ atree.Storable = SomeStorable{} +var _ atree.ContainerStorable = SomeStorable{} + +func (s SomeStorable) HasPointer() bool { + switch cs := s.Storable.(type) { + case atree.ContainerStorable: + return cs.HasPointer() + default: + return false + } +} + +func getSomeStorableEncodedPrefixSize(nestedLevels uint64) uint32 { + if nestedLevels == 1 { + return cborTagSize + } + return cborTagSize + someStorableWithMultipleNestedlevelsArraySize + getUintCBORSize(nestedLevels) +} func (s SomeStorable) ByteSize() uint32 { - return cborTagSize + s.Storable.ByteSize() + nonSomeStorable, nestedLevels := s.nonSomeStorable() + return getSomeStorableEncodedPrefixSize(nestedLevels) + nonSomeStorable.ByteSize() +} + +// nonSomeStorable returns a non-SomeStorable and nested levels of SomeStorable reached +// by traversing nested SomeStorable (SomeStorable containing SomeStorable, etc.) +// until it reaches a non-SomeStorable. +// For example, +// - `SomeStorable{true}` has non-SomeStorable `true`, and nested levels 1 +// - `SomeStorable{SomeStorable{1}}` has non-SomeStorable `1` and nested levels 2 +// - `SomeStorable{SomeStorable{[SomeStorable{SomeStorable{SomeStorable{1}}}]}} has +// non-SomeStorable `[SomeStorable{SomeStorable{SomeStorable{1}}}]` and nested levels 2 +func (s SomeStorable) nonSomeStorable() (atree.Storable, uint64) { + nestedLevels := uint64(1) + for { + switch storable := s.Storable.(type) { + case SomeStorable: + nestedLevels++ + s = storable + + default: + return storable, nestedLevels + } + } } func (s SomeStorable) StoredValue(storage atree.SlabStorage) (atree.Value, error) { @@ -19615,11 +19888,11 @@ func NewStorageReferenceValue( func (*StorageReferenceValue) isValue() {} -func (v *StorageReferenceValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *StorageReferenceValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitStorageReferenceValue(interpreter, v) } -func (*StorageReferenceValue) Walk(_ *Interpreter, _ func(Value)) { +func (*StorageReferenceValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP // NOTE: *not* walking referenced value! } @@ -19900,6 +20173,7 @@ func (v *StorageReferenceValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -19916,7 +20190,7 @@ func (v *StorageReferenceValue) Clone(_ *Interpreter) Value { ) } -func (*StorageReferenceValue) DeepRemove(_ *Interpreter) { +func (*StorageReferenceValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -19962,11 +20236,11 @@ func NewEphemeralReferenceValue( func (*EphemeralReferenceValue) isValue() {} -func (v *EphemeralReferenceValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *EphemeralReferenceValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitEphemeralReferenceValue(interpreter, v) } -func (*EphemeralReferenceValue) Walk(_ *Interpreter, _ func(Value)) { +func (*EphemeralReferenceValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP // NOTE: *not* walking referenced value! } @@ -20235,6 +20509,7 @@ func (v *EphemeralReferenceValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -20246,7 +20521,7 @@ func (v *EphemeralReferenceValue) Clone(_ *Interpreter) Value { return NewUnmeteredEphemeralReferenceValue(v.Authorized, v.Value, v.BorrowedType) } -func (*EphemeralReferenceValue) DeepRemove(_ *Interpreter) { +func (*EphemeralReferenceValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -20312,11 +20587,11 @@ var _ MemberAccessibleValue = AddressValue{} func (AddressValue) isValue() {} -func (v AddressValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v AddressValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitAddressValue(interpreter, v) } -func (AddressValue) Walk(_ *Interpreter, _ func(Value)) { +func (AddressValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -20452,6 +20727,7 @@ func (v AddressValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -20463,7 +20739,7 @@ func (v AddressValue) Clone(_ *Interpreter) Value { return v } -func (AddressValue) DeepRemove(_ *Interpreter) { +func (AddressValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -20540,11 +20816,11 @@ var _ MemberAccessibleValue = PathValue{} func (PathValue) isValue() {} -func (v PathValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v PathValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitPathValue(interpreter, v) } -func (PathValue) Walk(_ *Interpreter, _ func(Value)) { +func (PathValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -20738,6 +21014,7 @@ func (v PathValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -20749,7 +21026,7 @@ func (v PathValue) Clone(_ *Interpreter) Value { return v } -func (PathValue) DeepRemove(_ *Interpreter) { +func (PathValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -20807,11 +21084,11 @@ func (*PathCapabilityValue) isValue() {} func (*PathCapabilityValue) isCapabilityValue() {} -func (v *PathCapabilityValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *PathCapabilityValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitPathCapabilityValue(interpreter, v) } -func (v *PathCapabilityValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *PathCapabilityValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.Address) walkChild(v.Path) } @@ -20956,9 +21233,10 @@ func (v *PathCapabilityValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { if remove { - v.DeepRemove(interpreter) + v.DeepRemove(interpreter, hasNoParentContainer) interpreter.RemoveReferencedSlab(storable) } return v @@ -20972,9 +21250,9 @@ func (v *PathCapabilityValue) Clone(interpreter *Interpreter) Value { ) } -func (v *PathCapabilityValue) DeepRemove(interpreter *Interpreter) { - v.Address.DeepRemove(interpreter) - v.Path.DeepRemove(interpreter) +func (v *PathCapabilityValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { + v.Address.DeepRemove(interpreter, hasNoParentContainer) + v.Path.DeepRemove(interpreter, hasNoParentContainer) } func (v *PathCapabilityValue) ByteSize() uint32 { @@ -21033,11 +21311,11 @@ func (*IDCapabilityValue) isValue() {} func (*IDCapabilityValue) isCapabilityValue() {} -func (v *IDCapabilityValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *IDCapabilityValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitIDCapabilityValue(interpreter, v) } -func (v *IDCapabilityValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *IDCapabilityValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.ID) walkChild(v.Address) } @@ -21158,9 +21436,10 @@ func (v *IDCapabilityValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { if remove { - v.DeepRemove(interpreter) + v.DeepRemove(interpreter, hasNoParentContainer) interpreter.RemoveReferencedSlab(storable) } return v @@ -21174,8 +21453,8 @@ func (v *IDCapabilityValue) Clone(interpreter *Interpreter) Value { ) } -func (v *IDCapabilityValue) DeepRemove(interpreter *Interpreter) { - v.Address.DeepRemove(interpreter) +func (v *IDCapabilityValue) DeepRemove(interpreter *Interpreter, hasNoParentContainer bool) { + v.Address.DeepRemove(interpreter, hasNoParentContainer) } func (v *IDCapabilityValue) ByteSize() uint32 { @@ -21223,11 +21502,11 @@ func (PathLinkValue) isValue() {} func (PathLinkValue) isLinkValue() {} -func (v PathLinkValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v PathLinkValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitPathLinkValue(interpreter, v) } -func (v PathLinkValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v PathLinkValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.TargetPath) } @@ -21307,6 +21586,7 @@ func (v PathLinkValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -21321,7 +21601,7 @@ func (v PathLinkValue) Clone(interpreter *Interpreter) Value { } } -func (PathLinkValue) DeepRemove(_ *Interpreter) { +func (PathLinkValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -21362,7 +21642,7 @@ var _ EquatableValue = &PublishedValue{} func (*PublishedValue) isValue() {} -func (v *PublishedValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *PublishedValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitPublishedValue(interpreter, v) } @@ -21398,7 +21678,7 @@ func (v *PublishedValue) MeteredString(memoryGauge common.MemoryGauge, seenRefer ) } -func (v *PublishedValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *PublishedValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.Recipient) walkChild(v.Value) } @@ -21444,6 +21724,7 @@ func (v *PublishedValue) Transfer( remove bool, storable atree.Storable, preventTransfer map[atree.ValueID]struct{}, + hasNoParentContainer bool, ) Value { // NB: if the inner value of a PublishedValue can be a resource, // we must perform resource-related checks here as well @@ -21457,6 +21738,7 @@ func (v *PublishedValue) Transfer( remove, nil, preventTransfer, + hasNoParentContainer, ).(CapabilityValue) addressValue := v.Recipient.Transfer( @@ -21466,6 +21748,7 @@ func (v *PublishedValue) Transfer( remove, nil, preventTransfer, + hasNoParentContainer, ).(AddressValue) if remove { @@ -21486,7 +21769,7 @@ func (v *PublishedValue) Clone(interpreter *Interpreter) Value { } } -func (*PublishedValue) DeepRemove(_ *Interpreter) { +func (*PublishedValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -21529,11 +21812,11 @@ func (AccountLinkValue) isValue() {} func (AccountLinkValue) isLinkValue() {} -func (v AccountLinkValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v AccountLinkValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitAccountLinkValue(interpreter, v) } -func (AccountLinkValue) Walk(_ *Interpreter, _ func(Value)) { +func (AccountLinkValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -21606,6 +21889,7 @@ func (v AccountLinkValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -21617,7 +21901,7 @@ func (AccountLinkValue) Clone(_ *Interpreter) Value { return AccountLinkValue{} } -func (AccountLinkValue) DeepRemove(_ *Interpreter) { +func (AccountLinkValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_accountcapabilitycontroller.go b/runtime/interpreter/value_accountcapabilitycontroller.go index 0ef78b7b13..8bb4643ca8 100644 --- a/runtime/interpreter/value_accountcapabilitycontroller.go +++ b/runtime/interpreter/value_accountcapabilitycontroller.go @@ -87,11 +87,11 @@ func (v *AccountCapabilityControllerValue) CapabilityControllerBorrowType() Refe return v.BorrowType } -func (v *AccountCapabilityControllerValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *AccountCapabilityControllerValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitAccountCapabilityControllerValue(interpreter, v) } -func (v *AccountCapabilityControllerValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *AccountCapabilityControllerValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.CapabilityID) } @@ -178,6 +178,7 @@ func (v *AccountCapabilityControllerValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -192,7 +193,7 @@ func (v *AccountCapabilityControllerValue) Clone(_ *Interpreter) Value { } } -func (v *AccountCapabilityControllerValue) DeepRemove(_ *Interpreter) { +func (v *AccountCapabilityControllerValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_accountreference.go b/runtime/interpreter/value_accountreference.go index ea9923e0aa..69e959d55e 100644 --- a/runtime/interpreter/value_accountreference.go +++ b/runtime/interpreter/value_accountreference.go @@ -71,11 +71,11 @@ func (*AccountReferenceValue) isValue() {} func (*AccountReferenceValue) isReference() {} -func (v *AccountReferenceValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *AccountReferenceValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitAccountReferenceValue(interpreter, v) } -func (*AccountReferenceValue) Walk(_ *Interpreter, _ func(Value)) { +func (*AccountReferenceValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP // NOTE: *not* walking referenced value! } @@ -277,6 +277,7 @@ func (v *AccountReferenceValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -292,7 +293,7 @@ func (v *AccountReferenceValue) Clone(_ *Interpreter) Value { ) } -func (*AccountReferenceValue) DeepRemove(_ *Interpreter) { +func (*AccountReferenceValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_function.go b/runtime/interpreter/value_function.go index b29201bb8d..f79e36963e 100644 --- a/runtime/interpreter/value_function.go +++ b/runtime/interpreter/value_function.go @@ -96,11 +96,11 @@ func (f *InterpretedFunctionValue) MeteredString(memoryGauge common.MemoryGauge, return f.String() } -func (f *InterpretedFunctionValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (f *InterpretedFunctionValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitInterpretedFunctionValue(interpreter, f) } -func (f *InterpretedFunctionValue) Walk(_ *Interpreter, _ func(Value)) { +func (f *InterpretedFunctionValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -153,6 +153,7 @@ func (f *InterpretedFunctionValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { // TODO: actually not needed, value is not storable if remove { @@ -165,7 +166,7 @@ func (f *InterpretedFunctionValue) Clone(_ *Interpreter) Value { return f } -func (*InterpretedFunctionValue) DeepRemove(_ *Interpreter) { +func (*InterpretedFunctionValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -226,11 +227,11 @@ var _ ContractValue = &HostFunctionValue{} func (*HostFunctionValue) isValue() {} -func (f *HostFunctionValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (f *HostFunctionValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitHostFunctionValue(interpreter, f) } -func (f *HostFunctionValue) Walk(_ *Interpreter, _ func(Value)) { +func (f *HostFunctionValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -302,6 +303,7 @@ func (f *HostFunctionValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { // TODO: actually not needed, value is not storable if remove { @@ -314,7 +316,7 @@ func (f *HostFunctionValue) Clone(_ *Interpreter) Value { return f } -func (*HostFunctionValue) DeepRemove(_ *Interpreter) { +func (*HostFunctionValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } @@ -362,11 +364,11 @@ func (f BoundFunctionValue) MeteredString(memoryGauge common.MemoryGauge, seenRe return f.Function.MeteredString(memoryGauge, seenReferences) } -func (f BoundFunctionValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (f BoundFunctionValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitBoundFunctionValue(interpreter, f) } -func (f BoundFunctionValue) Walk(_ *Interpreter, _ func(Value)) { +func (f BoundFunctionValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -429,6 +431,7 @@ func (f BoundFunctionValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { // TODO: actually not needed, value is not storable if remove { @@ -441,6 +444,6 @@ func (f BoundFunctionValue) Clone(_ *Interpreter) Value { return f } -func (BoundFunctionValue) DeepRemove(_ *Interpreter) { +func (BoundFunctionValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_placeholder.go b/runtime/interpreter/value_placeholder.go index 4ab089e350..8f0648e0a9 100644 --- a/runtime/interpreter/value_placeholder.go +++ b/runtime/interpreter/value_placeholder.go @@ -45,11 +45,11 @@ func (f placeholderValue) MeteredString(_ common.MemoryGauge, _ SeenReferences) return "" } -func (f placeholderValue) Accept(_ *Interpreter, _ Visitor) { +func (f placeholderValue) Accept(_ *Interpreter, _ LocationRange, _ Visitor) { // NO-OP } -func (f placeholderValue) Walk(_ *Interpreter, _ func(Value)) { +func (f placeholderValue) Walk(_ *Interpreter, _ LocationRange, _ func(Value)) { // NO-OP } @@ -88,6 +88,7 @@ func (f placeholderValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { // TODO: actually not needed, value is not storable if remove { @@ -100,6 +101,6 @@ func (f placeholderValue) Clone(_ *Interpreter) Value { return f } -func (placeholderValue) DeepRemove(_ *Interpreter) { +func (placeholderValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_publickey.go b/runtime/interpreter/value_publickey.go index 9193ed3253..c346e36c37 100644 --- a/runtime/interpreter/value_publickey.go +++ b/runtime/interpreter/value_publickey.go @@ -72,6 +72,7 @@ func NewPublicKeyValue( false, nil, nil, + true, // publicKey is standalone because it is function parameter. ) }, } diff --git a/runtime/interpreter/value_storagecapabilitycontroller.go b/runtime/interpreter/value_storagecapabilitycontroller.go index 8927a2820c..9bf6140f69 100644 --- a/runtime/interpreter/value_storagecapabilitycontroller.go +++ b/runtime/interpreter/value_storagecapabilitycontroller.go @@ -107,11 +107,11 @@ func (v *StorageCapabilityControllerValue) CapabilityControllerBorrowType() Refe return v.BorrowType } -func (v *StorageCapabilityControllerValue) Accept(interpreter *Interpreter, visitor Visitor) { +func (v *StorageCapabilityControllerValue) Accept(interpreter *Interpreter, _ LocationRange, visitor Visitor) { visitor.VisitStorageCapabilityControllerValue(interpreter, v) } -func (v *StorageCapabilityControllerValue) Walk(_ *Interpreter, walkChild func(Value)) { +func (v *StorageCapabilityControllerValue) Walk(_ *Interpreter, _ LocationRange, walkChild func(Value)) { walkChild(v.TargetPath) walkChild(v.CapabilityID) } @@ -202,6 +202,7 @@ func (v *StorageCapabilityControllerValue) Transfer( remove bool, storable atree.Storable, _ map[atree.ValueID]struct{}, + _ bool, ) Value { if remove { interpreter.RemoveReferencedSlab(storable) @@ -217,7 +218,7 @@ func (v *StorageCapabilityControllerValue) Clone(interpreter *Interpreter) Value } } -func (v *StorageCapabilityControllerValue) DeepRemove(_ *Interpreter) { +func (v *StorageCapabilityControllerValue) DeepRemove(_ *Interpreter, _ bool) { // NO-OP } diff --git a/runtime/interpreter/value_test.go b/runtime/interpreter/value_test.go index dd8e35b2e6..d73c86f666 100644 --- a/runtime/interpreter/value_test.go +++ b/runtime/interpreter/value_test.go @@ -176,6 +176,7 @@ func TestOwnerArrayDeepCopy(t *testing.T) { false, nil, nil, + true, // array is standalone. ) array = arrayCopy.(*ArrayValue) @@ -570,6 +571,7 @@ func TestOwnerDictionaryCopy(t *testing.T) { false, nil, nil, + true, // dictionary is standalone. ) dictionaryCopy := copyResult.(*DictionaryValue) @@ -874,6 +876,7 @@ func TestOwnerCompositeCopy(t *testing.T) { false, nil, nil, + true, // composite is standalone. ).(*CompositeValue) value = composite.GetMember( @@ -1234,7 +1237,7 @@ func TestVisitor(t *testing.T) { common.ZeroAddress, ) - value.Accept(inter, visitor) + value.Accept(inter, EmptyLocationRange, visitor) require.Equal(t, 1, intVisits) require.Equal(t, 1, stringVisits) diff --git a/runtime/interpreter/walk.go b/runtime/interpreter/walk.go index 626673e130..b5349acac1 100644 --- a/runtime/interpreter/walk.go +++ b/runtime/interpreter/walk.go @@ -37,9 +37,13 @@ func WalkValue(interpreter *Interpreter, walker ValueWalker, value Value) { return } - value.Walk(interpreter, func(child Value) { - WalkValue(interpreter, walker, child) - }) + value.Walk( + interpreter, + EmptyLocationRange, + func(child Value) { + WalkValue(interpreter, walker, child) + }, + ) walker.WalkValue(interpreter, nil) } diff --git a/runtime/runtime_memory_metering_test.go b/runtime/runtime_memory_metering_test.go index 196e4c0d77..e754593062 100644 --- a/runtime/runtime_memory_metering_test.go +++ b/runtime/runtime_memory_metering_test.go @@ -1108,7 +1108,7 @@ func TestMeterEncoding(t *testing.T) { ) require.NoError(t, err) - assert.Equal(t, 87, int(meter.getMemory(common.MemoryKindBytes))) + assert.Equal(t, 75, int(meter.getMemory(common.MemoryKindBytes))) }) t.Run("string in loop", func(t *testing.T) { @@ -1156,7 +1156,7 @@ func TestMeterEncoding(t *testing.T) { ) require.NoError(t, err) - assert.Equal(t, 62787, int(meter.getMemory(common.MemoryKindBytes))) + assert.Equal(t, 61455, int(meter.getMemory(common.MemoryKindBytes))) }) t.Run("composite", func(t *testing.T) { @@ -1206,6 +1206,6 @@ func TestMeterEncoding(t *testing.T) { ) require.NoError(t, err) - assert.Equal(t, 76941, int(meter.getMemory(common.MemoryKindBytes))) + assert.Equal(t, 58323, int(meter.getMemory(common.MemoryKindBytes))) }) } diff --git a/runtime/runtime_test.go b/runtime/runtime_test.go index c64d36d17b..51f64023d2 100644 --- a/runtime/runtime_test.go +++ b/runtime/runtime_test.go @@ -5801,10 +5801,11 @@ func TestRuntimeContractWriteback(t *testing.T) { assert.Equal(t, []ownerKeyPair{ - // contract value + // Storage map is modified because contract value is inlined in contract storage map. + // NOTE: contract value slab doesn't exist. { addressValue[:], - []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, }, }, writes, @@ -5900,6 +5901,7 @@ func TestRuntimeStorageWriteback(t *testing.T) { []byte("contract"), }, // contract value + // NOTE: contract value slab is empty because it is inlined in contract domain storage map { addressValue[:], []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, @@ -5943,11 +5945,13 @@ func TestRuntimeStorageWriteback(t *testing.T) { []byte("storage"), }, // resource value + // NOTE: resource value slab is empty because it is inlined in storage domain storage map { addressValue[:], []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3}, }, // storage domain storage map + // NOTE: resource value slab is inlined. { addressValue[:], []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4}, @@ -6009,10 +6013,11 @@ func TestRuntimeStorageWriteback(t *testing.T) { assert.Equal(t, []ownerKeyPair{ - // resource value + // Storage map is modified because resource value is inlined in storage map + // NOTE: resource value slab is empty. { addressValue[:], - []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3}, + []byte{'$', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4}, }, }, writes, @@ -7579,7 +7584,7 @@ func TestRuntimeComputationMetring(t *testing.T) { `, ok: true, hits: 3, - intensity: 88, + intensity: 76, }, } diff --git a/runtime/stdlib/account.go b/runtime/stdlib/account.go index 0fcf70d38c..6eb298f6e4 100644 --- a/runtime/stdlib/account.go +++ b/runtime/stdlib/account.go @@ -1049,6 +1049,7 @@ func newAuthAccountInboxPublishFunction( true, nil, nil, + true, // New PublishedValue is standalone. ) storageMapKey := interpreter.StringStorageMapKey(nameValue.Str) @@ -1116,6 +1117,7 @@ func newAuthAccountInboxUnpublishFunction( true, nil, nil, + false, // publishedValue is an element in storage map because it is returned by ReadStored. ) inter.WriteStored( @@ -1202,6 +1204,7 @@ func newAuthAccountInboxClaimFunction( true, nil, nil, + false, // publishedValue is an element in storage map because it is returned by ReadStored. ) inter.WriteStored( @@ -3263,6 +3266,7 @@ func newAuthAccountCapabilitiesPublishFunction( true, nil, nil, + true, // capabilityValue is standalone because it is from invocation.Arguments[0]. ).(*interpreter.IDCapabilityValue) if !ok { panic(errors.NewUnreachableError()) @@ -3333,6 +3337,7 @@ func newAuthAccountCapabilitiesUnpublishFunction( true, nil, nil, + false, // capabilityValue is an element of storage map. ).(*interpreter.IDCapabilityValue) if !ok { panic(errors.NewUnreachableError()) @@ -3480,6 +3485,7 @@ func newAuthAccountCapabilitiesMigrateLinkFunction( true, nil, nil, + true, // capabilityValue is standalone. ).(*interpreter.IDCapabilityValue) if !ok { panic(errors.NewUnreachableError()) diff --git a/runtime/storage.go b/runtime/storage.go index e9982516a5..a35558974c 100644 --- a/runtime/storage.go +++ b/runtime/storage.go @@ -49,6 +49,7 @@ func NewStorage(ledger atree.Ledger, memoryGauge common.MemoryGauge) *Storage { decodeStorable := func( decoder *cbor.StreamDecoder, slabID atree.SlabID, + inlinedExtraData []atree.ExtraData, ) ( atree.Storable, error, @@ -56,6 +57,7 @@ func NewStorage(ledger atree.Ledger, memoryGauge common.MemoryGauge) *Storage { return interpreter.DecodeStorable( decoder, slabID, + inlinedExtraData, memoryGauge, ) } diff --git a/runtime/storage_test.go b/runtime/storage_test.go index b405a3e100..0ca1bbb5f3 100644 --- a/runtime/storage_test.go +++ b/runtime/storage_test.go @@ -1736,11 +1736,14 @@ func TestRuntimeStorageTransfer(t *testing.T) { nonEmptyKeys++ } } - // 5: + + // TODO: maybe retrieve and compare stored values from 2 accounts + + // 4: + // NOTE: with atree inlining, array is inlined inside storage map // - 2x storage index for storage domain storage map // - 2x storage domain storage map - // - array (atree array) - assert.Equal(t, 5, nonEmptyKeys) + assert.Equal(t, 4, nonEmptyKeys) } func TestRuntimeResourceOwnerChange(t *testing.T) { @@ -1900,18 +1903,16 @@ func TestRuntimeResourceOwnerChange(t *testing.T) { assert.Equal(t, []string{ // account 0x1: + // NOTE: with atree inlining, contract is inlined in contract map // storage map (domain key + map slab) - // + contract map (domain key + map slap) - // + contract - "\x00\x00\x00\x00\x00\x00\x00\x01|$\x00\x00\x00\x00\x00\x00\x00\x01", + // + contract map (domain key + map slab) "\x00\x00\x00\x00\x00\x00\x00\x01|$\x00\x00\x00\x00\x00\x00\x00\x02", "\x00\x00\x00\x00\x00\x00\x00\x01|$\x00\x00\x00\x00\x00\x00\x00\x04", "\x00\x00\x00\x00\x00\x00\x00\x01|contract", "\x00\x00\x00\x00\x00\x00\x00\x01|storage", // account 0x2 + // NOTE: with atree inlining, resource is inlined in storage map // storage map (domain key + map slab) - // + resource - "\x00\x00\x00\x00\x00\x00\x00\x02|$\x00\x00\x00\x00\x00\x00\x00\x01", "\x00\x00\x00\x00\x00\x00\x00\x02|$\x00\x00\x00\x00\x00\x00\x00\x02", "\x00\x00\x00\x00\x00\x00\x00\x02|storage", }, diff --git a/runtime/tests/interpreter/array_test.go b/runtime/tests/interpreter/array_test.go index 03ed464bdb..ed0e428aff 100644 --- a/runtime/tests/interpreter/array_test.go +++ b/runtime/tests/interpreter/array_test.go @@ -35,13 +35,17 @@ func dictionaryKeyValues(inter *interpreter.Interpreter, dict *interpreter.Dicti count := dict.Count() * 2 result := make([]interpreter.Value, count) i := 0 - dict.Iterate(inter, func(key, value interpreter.Value) (resume bool) { - result[i*2] = key - result[i*2+1] = value - i++ + dict.Iterate( + inter, + interpreter.EmptyLocationRange, + func(key, value interpreter.Value) (resume bool) { + result[i*2] = key + result[i*2+1] = value + i++ - return true - }) + return true + }, + ) return result } @@ -65,23 +69,27 @@ func dictionaryEntries[K, V any]( iterStatus := true idx := 0 - dict.Iterate(inter, func(rawKey, rawValue interpreter.Value) (resume bool) { - key, ok := fromKey(rawKey) + dict.Iterate( + inter, + interpreter.EmptyLocationRange, + func(rawKey, rawValue interpreter.Value) (resume bool) { + key, ok := fromKey(rawKey) - if !ok { - iterStatus = false - return iterStatus - } + if !ok { + iterStatus = false + return iterStatus + } - value, ok := fromVal(rawValue) - if !ok { - iterStatus = false - return iterStatus - } + value, ok := fromVal(rawValue) + if !ok { + iterStatus = false + return iterStatus + } - res[idx] = entry[K, V]{key, value} - return iterStatus - }) + res[idx] = entry[K, V]{key, value} + return iterStatus + }, + ) return res, iterStatus } diff --git a/runtime/tests/interpreter/interpreter_test.go b/runtime/tests/interpreter/interpreter_test.go index f6f915f6a5..4dfced3922 100644 --- a/runtime/tests/interpreter/interpreter_test.go +++ b/runtime/tests/interpreter/interpreter_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/onflow/cadence/runtime" - "github.com/onflow/cadence/runtime/activations" - "github.com/onflow/atree" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/activations" "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" @@ -200,8 +200,10 @@ func parseCheckAndInterpretWithOptionsAndMemoryMetering( config = *options.Config } config.InvalidatedResourceValidationEnabled = true - config.AtreeValueValidationEnabled = true - config.AtreeStorageValidationEnabled = true + if memoryGauge == nil { + config.AtreeValueValidationEnabled = true + config.AtreeStorageValidationEnabled = true + } if config.UUIDHandler == nil { config.UUIDHandler = func() (uint64, error) { uuid++ @@ -5249,6 +5251,7 @@ func TestInterpretReferenceFailableDowncasting(t *testing.T) { true, nil, nil, + true, // r is standalone. ) domain := storagePath.Domain.Identifier() @@ -8121,6 +8124,7 @@ func TestInterpretResourceMovingAndBorrowing(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") @@ -8167,7 +8171,8 @@ func TestInterpretResourceMovingAndBorrowing(t *testing.T) { permanentSlabs = append(permanentSlabs, slab) } - require.Equal(t, 2, len(permanentSlabs)) + // permanent slab is R1 (R2 is inlined in R1 slab) + require.Equal(t, 1, len(permanentSlabs)) sort.Slice(permanentSlabs, func(i, j int) bool { a := permanentSlabs[i].SlabID() @@ -8185,7 +8190,6 @@ func TestInterpretResourceMovingAndBorrowing(t *testing.T) { require.Equal(t, []string{ `S.test.R1(r2: S.test.R2(value: "test", uuid: 2), uuid: 1)`, - `S.test.R2(value: "test", uuid: 2)`, }, storedValues, ) diff --git a/runtime/tests/interpreter/memory_metering_test.go b/runtime/tests/interpreter/memory_metering_test.go index b1d9912ba1..22d406d208 100644 --- a/runtime/tests/interpreter/memory_metering_test.go +++ b/runtime/tests/interpreter/memory_metering_test.go @@ -71,12 +71,12 @@ func TestInterpretArrayMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int8] = [] - let y: [[String]] = [[]] - let z: [[[Bool]]] = [[[]]] - } -` + pub fun main() { + let x: [Int8] = [] + let y: [[String]] = [[]] + let z: [[[Bool]]] = [[[]]] + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -94,10 +94,7 @@ func TestInterpretArrayMetering(t *testing.T) { // 2 String: 1 for type, 1 for value // 3 Bool: 1 for type, 2 for value assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - // 0 for `x` - // 1 for `y` - // 4 for `z` - assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindVariableSizedStaticType)) + assert.Equal(t, uint64(10), meter.getMemory(common.MemoryKindVariableSizedStaticType)) }) t.Run("iteration", func(t *testing.T) { @@ -126,8 +123,7 @@ func TestInterpretArrayMetering(t *testing.T) { // 4 Int8: 1 for type, 3 for values assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - // 3: 1 for each [] in `values` - assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindVariableSizedStaticType)) + assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindVariableSizedStaticType)) }) t.Run("contains", func(t *testing.T) { @@ -258,12 +254,12 @@ func TestInterpretArrayMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int8] = [] - x.insert(at:0, 3) - x.insert(at:1, 3) - } -` + pub fun main() { + let x: [Int8] = [] + x.insert(at:0, 3) + x.insert(at:1, 3) + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -275,19 +271,19 @@ func TestInterpretArrayMetering(t *testing.T) { assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindAtreeArrayMetaDataSlab)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeArrayElementOverhead)) assert.Equal(t, uint64(7), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindVariableSizedStaticType)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindVariableSizedStaticType)) }) t.Run("update", func(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int128] = [0, 1, 2, 3] // uses 2 data slabs and 1 metadata slab - x[0] = 1 // adds 1 data and 1 metadata slab - x[2] = 1 // adds 1 data and 1 metadata slab - } -` + pub fun main() { + let x: [Int128] = [0, 1, 2, 3] // uses 2 data slabs and 1 metadata slab + x[0] = 1 // adds 1 data and 1 metadata slab + x[2] = 1 // adds 1 data and 1 metadata slab + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -304,11 +300,11 @@ func TestInterpretArrayMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int128] = [0, 1, 2] // uses 2 data slabs and 1 metadata slab - x[0] = 1 // fits in existing slab - x[2] = 1 // fits in existing slab - } + pub fun main() { + let x: [Int128] = [0, 1, 2] // uses 2 data slabs and 1 metadata slab + x[0] = 1 // fits in existing slab + x[2] = 1 // fits in existing slab + } ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -326,15 +322,15 @@ func TestInterpretArrayMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int8; 0] = [] - let y: [Int8; 1] = [2] - let z: [Int8; 2] = [2, 4] - let w: [[Int8; 2]] = [[2, 4]] - let r: [[Int8; 2]] = [[2, 4], [8, 16]] - let q: [[Int8; 2]; 2] = [[2, 4], [8, 16]] - } -` + pub fun main() { + let x: [Int8; 0] = [] + let y: [Int8; 1] = [2] + let z: [Int8; 2] = [2, 4] + let w: [[Int8; 2]] = [[2, 4]] + let r: [[Int8; 2]] = [[2, 4], [8, 16]] + let q: [[Int8; 2]; 2] = [[2, 4], [8, 16]] + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -346,10 +342,7 @@ func TestInterpretArrayMetering(t *testing.T) { assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindAtreeArrayMetaDataSlab)) assert.Equal(t, uint64(66), meter.getMemory(common.MemoryKindAtreeArrayElementOverhead)) - // 1 for `w`: 1 for the element - // 2 for `r`: 1 for each element - // 2 for `q`: 1 for each element - assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindConstantSizedStaticType)) + assert.Equal(t, uint64(12), meter.getMemory(common.MemoryKindConstantSizedStaticType)) // 2 for `q` type // 1 for each other type assert.Equal(t, uint64(7), meter.getMemory(common.MemoryKindConstantSizedType)) @@ -359,16 +352,16 @@ func TestInterpretArrayMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: [Int128] = [] // 2 data slabs - x.insert(at:0, 3) // fits in existing slab - x.insert(at:1, 3) // fits in existing slab - x.insert(at:2, 3) // adds 1 metadata and data slab - x.insert(at:3, 3) // fits in existing slab - x.insert(at:4, 3) // adds 1 data slab - x.insert(at:5, 3) // fits in existing slab - } -` + pub fun main() { + let x: [Int128] = [] // 2 data slabs + x.insert(at:0, 3) // fits in existing slab + x.insert(at:1, 3) // fits in existing slab + x.insert(at:2, 3) // adds 1 metadata and data slab + x.insert(at:3, 3) // fits in existing slab + x.insert(at:4, 3) // adds 1 data slab + x.insert(at:5, 3) // fits in existing slab + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -385,7 +378,7 @@ func TestInterpretArrayMetering(t *testing.T) { // 2 Int8 for `r` elements // 2 Int8 for `q` elements assert.Equal(t, uint64(19), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindVariableSizedStaticType)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindVariableSizedStaticType)) }) } @@ -396,11 +389,11 @@ func TestInterpretDictionaryMetering(t *testing.T) { t.Parallel() script := ` - pub fun main() { - let x: {Int8: String} = {} - let y: {String: {Int8: String}} = {"a": {}} - } - ` + pub fun main() { + let x: {Int8: String} = {} + let y: {String: {Int8: String}} = {"a": {}} + } + ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -408,7 +401,6 @@ func TestInterpretDictionaryMetering(t *testing.T) { _, err := inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindStringValue)) assert.Equal(t, uint64(9), meter.getMemory(common.MemoryKindDictionaryValueBase)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeMapElementOverhead)) assert.Equal(t, uint64(9), meter.getMemory(common.MemoryKindAtreeMapDataSlab)) @@ -416,12 +408,7 @@ func TestInterpretDictionaryMetering(t *testing.T) { assert.Equal(t, uint64(159), meter.getMemory(common.MemoryKindAtreeMapPreAllocatedElement)) assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindVariable)) assert.Equal(t, uint64(9), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - // 1 for `x` - // 7 for `y`: 2 for type, 5 for value - // Note that the number of static types allocated raises 1 with each value. - // 1, 2, 3, ... elements each use 5, 6, 7, ... static types. - // This is cumulative so 3 elements allocate 5+6+7=18 static types. - assert.Equal(t, uint64(8), meter.getMemory(common.MemoryKindDictionaryStaticType)) + assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindDictionaryStaticType)) }) t.Run("iteration", func(t *testing.T) { @@ -448,9 +435,7 @@ func TestInterpretDictionaryMetering(t *testing.T) { // 4 Int8: 1 for type, 3 for values // 4 String: 1 for type, 3 for values assert.Equal(t, uint64(8), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - // 1 for type - // 6: 2 for each element - assert.Equal(t, uint64(7), meter.getMemory(common.MemoryKindDictionaryStaticType)) + assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindDictionaryStaticType)) assert.Equal(t, uint64(480), meter.getMemory(common.MemoryKindAtreeMapPreAllocatedElement)) }) @@ -479,7 +464,7 @@ func TestInterpretDictionaryMetering(t *testing.T) { script := ` pub fun main() { - let x: {Int8: String} = {} + let x: {Int8: String} = {} x.insert(key: 5, "") x.insert(key: 4, "") } @@ -496,7 +481,7 @@ func TestInterpretDictionaryMetering(t *testing.T) { assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeMapDataSlab)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindAtreeMapMetaDataSlab)) assert.Equal(t, uint64(10), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindDictionaryStaticType)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindDictionaryStaticType)) assert.Equal(t, uint64(32), meter.getMemory(common.MemoryKindAtreeMapPreAllocatedElement)) }) @@ -538,13 +523,13 @@ func TestInterpretDictionaryMetering(t *testing.T) { pub fun main() { let x: {Int8: Int8} = {} // 2 data slabs x.insert(key: 0, 0) // all fit in slab - x.insert(key: 1, 1) - x.insert(key: 2, 2) - x.insert(key: 3, 3) - x.insert(key: 4, 4) - x.insert(key: 5, 5) + x.insert(key: 1, 1) + x.insert(key: 2, 2) + x.insert(key: 3, 3) + x.insert(key: 4, 4) + x.insert(key: 5, 5) x.insert(key: 6, 6) - x.insert(key: 7, 7) + x.insert(key: 7, 7) x.insert(key: 8, 8) } ` @@ -643,8 +628,7 @@ func TestInterpretCompositeMetering(t *testing.T) { _, err := inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindStringValue)) - assert.Equal(t, uint64(66), meter.getMemory(common.MemoryKindRawString)) + assert.Equal(t, uint64(9), meter.getMemory(common.MemoryKindRawString)) assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindCompositeValueBase)) assert.Equal(t, uint64(5), meter.getMemory(common.MemoryKindAtreeMapDataSlab)) assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindAtreeMapMetaDataSlab)) @@ -652,7 +636,7 @@ func TestInterpretCompositeMetering(t *testing.T) { assert.Equal(t, uint64(32), meter.getMemory(common.MemoryKindAtreeMapPreAllocatedElement)) assert.Equal(t, uint64(8), meter.getMemory(common.MemoryKindVariable)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindCompositeStaticType)) - assert.Equal(t, uint64(9), meter.getMemory(common.MemoryKindCompositeTypeInfo)) + assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindCompositeTypeInfo)) assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindCompositeField)) assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindInvocation)) }) @@ -684,7 +668,7 @@ func TestInterpretCompositeMetering(t *testing.T) { assert.Equal(t, uint64(7), meter.getMemory(common.MemoryKindVariable)) assert.Equal(t, uint64(7), meter.getMemory(common.MemoryKindCompositeStaticType)) - assert.Equal(t, uint64(24), meter.getMemory(common.MemoryKindCompositeTypeInfo)) + assert.Equal(t, uint64(21), meter.getMemory(common.MemoryKindCompositeTypeInfo)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindCompositeField)) assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindInvocation)) }) @@ -697,9 +681,7 @@ func TestInterpretSimpleCompositeMetering(t *testing.T) { t.Parallel() script := ` - pub fun main(a: AuthAccount) { - - } + pub fun main(a: AuthAccount) {} ` meter := newTestMemoryGauge() @@ -717,9 +699,7 @@ func TestInterpretSimpleCompositeMetering(t *testing.T) { t.Parallel() script := ` - pub fun main(a: PublicAccount) { - - } + pub fun main(a: PublicAccount) {} ` meter := newTestMemoryGauge() @@ -781,7 +761,7 @@ func TestInterpretCompositeFieldMetering(t *testing.T) { _, err := inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(16), meter.getMemory(common.MemoryKindRawString)) + assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindRawString)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindCompositeValueBase)) assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindAtreeMapElementOverhead)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeMapDataSlab)) @@ -812,7 +792,7 @@ func TestInterpretCompositeFieldMetering(t *testing.T) { _, err := inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(34), meter.getMemory(common.MemoryKindRawString)) + assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindRawString)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeMapDataSlab)) assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAtreeMapElementOverhead)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindAtreeMapMetaDataSlab)) @@ -1314,7 +1294,7 @@ func TestInterpretOptionalValueMetering(t *testing.T) { assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindOptionalValue)) assert.Equal(t, uint64(14), meter.getMemory(common.MemoryKindPrimitiveStaticType)) - assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindDictionaryStaticType)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindDictionaryStaticType)) }) t.Run("dictionary set", func(t *testing.T) { @@ -7001,8 +6981,8 @@ func TestInterpretPathCapabilityValueMetering(t *testing.T) { require.NoError(t, err) assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindPathCapabilityValue)) - assert.Equal(t, uint64(4), meter.getMemory(common.MemoryKindPathValue)) - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindReferenceStaticType)) + assert.Equal(t, uint64(3), meter.getMemory(common.MemoryKindPathValue)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindReferenceStaticType)) }) t.Run("array element", func(t *testing.T) { @@ -7053,8 +7033,8 @@ func TestInterpretPathLinkValueMetering(t *testing.T) { require.NoError(t, err) // Metered twice only when Atree validation is enabled. - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindPathLinkValue)) - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindReferenceStaticType)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindPathLinkValue)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindReferenceStaticType)) }) } @@ -7091,7 +7071,7 @@ func TestInterpretAccountLinkValueMetering(t *testing.T) { require.NoError(t, err) // Metered twice only when Atree validation is enabled. - assert.Equal(t, uint64(2), meter.getMemory(common.MemoryKindAccountLinkValue)) + assert.Equal(t, uint64(1), meter.getMemory(common.MemoryKindAccountLinkValue)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindReferenceStaticType)) }) } @@ -7163,9 +7143,7 @@ func TestInterpretVariableMetering(t *testing.T) { var a = 3 let b = false - pub fun main() { - - } + pub fun main() {} ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -7180,9 +7158,7 @@ func TestInterpretVariableMetering(t *testing.T) { t.Parallel() script := ` - pub fun main(a: String, b: Bool) { - - } + pub fun main(a: String, b: Bool) {} ` meter := newTestMemoryGauge() inter := parseCheckAndInterpretWithMemoryMetering(t, script, meter) @@ -7607,7 +7583,7 @@ func TestInterpretUFix64Metering(t *testing.T) { script := ` pub fun main() { - let x: UFix64 = 2.5 - 1.4 + let x: UFix64 = 2.5 - 1.4 } ` diff --git a/runtime/tests/interpreter/reference_test.go b/runtime/tests/interpreter/reference_test.go index 34abede312..914971b4a9 100644 --- a/runtime/tests/interpreter/reference_test.go +++ b/runtime/tests/interpreter/reference_test.go @@ -744,6 +744,7 @@ func TestInterpretReferenceUseAfterShiftStatementMove(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") diff --git a/runtime/tests/interpreter/resources_test.go b/runtime/tests/interpreter/resources_test.go index ba6069c2d7..2391dd17ff 100644 --- a/runtime/tests/interpreter/resources_test.go +++ b/runtime/tests/interpreter/resources_test.go @@ -196,6 +196,7 @@ func TestInterpretImplicitResourceRemovalFromContainer(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") @@ -325,6 +326,7 @@ func TestInterpretImplicitResourceRemovalFromContainer(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") @@ -449,6 +451,7 @@ func TestInterpretImplicitResourceRemovalFromContainer(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") @@ -578,6 +581,7 @@ func TestInterpretImplicitResourceRemovalFromContainer(t *testing.T) { false, nil, nil, + true, // r1 is standalone. ) r1Type := checker.RequireGlobalType(t, inter.Program.Elaboration, "R1") diff --git a/runtime/tests/interpreter/values_test.go b/runtime/tests/interpreter/values_test.go index 99436e7f80..2c1519dfb2 100644 --- a/runtime/tests/interpreter/values_test.go +++ b/runtime/tests/interpreter/values_test.go @@ -42,11 +42,11 @@ import ( // TODO: make these program args? const containerMaxDepth = 3 const containerMaxSize = 100 -const innerContainerMaxSize = 300 const compositeMaxFields = 10 var runSmokeTests = flag.Bool("runSmokeTests", false, "Run smoke tests on values") var validateAtree = flag.Bool("validateAtree", true, "Enable atree validation") +var smokeTestSeed = flag.Int64("smokeTestSeed", -1, "Seed for prng (-1 specifies current Unix time)") func TestRandomMapOperations(t *testing.T) { if !*runSmokeTests { @@ -131,13 +131,17 @@ func TestRandomMapOperations(t *testing.T) { t.Run("iterate", func(t *testing.T) { require.Equal(t, testMap.Count(), entries.size()) - testMap.Iterate(inter, func(key, value interpreter.Value) (resume bool) { - orgValue, ok := entries.get(inter, key) - require.True(t, ok, "cannot find key: %v", key) + testMap.Iterate( + inter, + interpreter.EmptyLocationRange, + func(key, value interpreter.Value) (resume bool) { + orgValue, ok := entries.get(inter, key) + require.True(t, ok, "cannot find key: %v", key) - utils.AssertValuesEqual(t, inter, orgValue, value) - return true - }) + utils.AssertValuesEqual(t, inter, orgValue, value) + return true + }, + ) }) t.Run("deep copy", func(t *testing.T) { @@ -149,6 +153,7 @@ func TestRandomMapOperations(t *testing.T) { false, nil, nil, + true, // testMap is standalone. ).(*interpreter.DictionaryValue) require.Equal(t, entries.size(), copyOfTestMap.Count()) @@ -169,7 +174,7 @@ func TestRandomMapOperations(t *testing.T) { }) t.Run("deep remove", func(t *testing.T) { - copyOfTestMap.DeepRemove(inter) + copyOfTestMap.DeepRemove(inter, true) err = storage.Remove(copyOfTestMap.SlabID()) require.NoError(t, err) @@ -489,6 +494,7 @@ func TestRandomMapOperations(t *testing.T) { true, nil, nil, + true, // dictionary is standalone. ).(*interpreter.DictionaryValue) require.Equal(t, entries.size(), movedDictionary.Count()) @@ -604,6 +610,7 @@ func TestRandomArrayOperations(t *testing.T) { false, nil, nil, + true, // testArray is standalone. ).(*interpreter.ArrayValue) require.Equal(t, len(elements), copyOfTestArray.Count()) @@ -618,7 +625,7 @@ func TestRandomArrayOperations(t *testing.T) { }) t.Run("deep removal", func(t *testing.T) { - copyOfTestArray.DeepRemove(inter) + copyOfTestArray.DeepRemove(inter, true) err = storage.Remove(copyOfTestArray.SlabID()) require.NoError(t, err) @@ -860,6 +867,7 @@ func TestRandomArrayOperations(t *testing.T) { true, nil, nil, + true, // array is standalone. ).(*interpreter.ArrayValue) require.Equal(t, len(elements), movedArray.Count()) @@ -951,6 +959,7 @@ func TestRandomCompositeValueOperations(t *testing.T) { false, nil, nil, + true, // testComposite is standalone. ).(*interpreter.CompositeValue) for name, orgValue := range orgFields { @@ -963,7 +972,7 @@ func TestRandomCompositeValueOperations(t *testing.T) { }) t.Run("deep remove", func(t *testing.T) { - copyOfTestComposite.DeepRemove(inter) + copyOfTestComposite.DeepRemove(inter, true) err = storage.Remove(copyOfTestComposite.SlabID()) require.NoError(t, err) @@ -992,6 +1001,7 @@ func TestRandomCompositeValueOperations(t *testing.T) { false, nil, nil, + true, // testComposite is standalone. ).(*interpreter.CompositeValue) require.NoError(t, err) @@ -1017,6 +1027,7 @@ func TestRandomCompositeValueOperations(t *testing.T) { true, nil, nil, + true, // composite is standalone. ).(*interpreter.CompositeValue) // Cleanup the slab of original composite. @@ -1135,7 +1146,10 @@ type randomValueGenerator struct { } func newRandomValueGenerator() randomValueGenerator { - seed := time.Now().UnixNano() + seed := *smokeTestSeed + if seed == -1 { + seed = time.Now().UnixNano() + } return randomValueGenerator{ seed: seed, @@ -1370,7 +1384,7 @@ func (r randomValueGenerator) randomDictionaryValue( currentDepth int, ) interpreter.Value { - entryCount := r.randomInt(innerContainerMaxSize) + entryCount := r.randomInt(containerMaxSize) keyValues := make([]interpreter.Value, entryCount*2) for i := 0; i < entryCount; i++ { @@ -1397,7 +1411,7 @@ func (r randomValueGenerator) randomInt(upperBound int) int { } func (r randomValueGenerator) randomArrayValue(inter *interpreter.Interpreter, currentDepth int) interpreter.Value { - elementsCount := r.randomInt(innerContainerMaxSize) + elementsCount := r.randomInt(containerMaxSize) elements := make([]interpreter.Value, elementsCount) for i := 0; i < elementsCount; i++ { @@ -1615,3 +1629,231 @@ func (m *valueMap) internalKey(inter *interpreter.Interpreter, key interpreter.V func (m *valueMap) size() int { return len(m.keys) } + +// This test is a reproducer for "slab was not reachable from leaves" false alarm. +// https://github.com/onflow/cadence/pull/2882#issuecomment-1781298107 +// In this test, storage.CheckHealth() should be called after array.DeepRemove(), +// not in the middle of array.DeepRemove(). +// CheckHealth() is called in the middle of array.DeepRemove() when: +// - array.DeepRemove() calls childArray1 and childArray2 DeepRemove() +// - DeepRemove() calls maybeValidateAtreeValue() +// - maybeValidateAtreeValue() calls CheckHealth() +func TestCheckStorageHealthInMiddleOfDeepRemove(t *testing.T) { + + storage := newUnmeteredInMemoryStorage() + inter, err := interpreter.NewInterpreter( + &interpreter.Program{ + Program: ast.NewProgram(nil, []ast.Declaration{}), + Elaboration: sema.NewElaboration(nil), + }, + utils.TestLocation, + &interpreter.Config{ + Storage: storage, + ImportLocationHandler: func(inter *interpreter.Interpreter, location common.Location) interpreter.Import { + return interpreter.VirtualImport{ + Elaboration: inter.Program.Elaboration, + } + }, + AtreeStorageValidationEnabled: true, + AtreeValueValidationEnabled: true, + }, + ) + require.NoError(t, err) + + owner := common.Address{'A'} + + // Create a small child array which will be inlined in parent container. + childArray1 := interpreter.NewArrayValue( + inter, + interpreter.EmptyLocationRange, + interpreter.VariableSizedStaticType{ + Type: interpreter.PrimitiveStaticTypeAnyStruct, + }, + owner, + interpreter.NewUnmeteredStringValue("a"), + ) + + size := int(atree.MaxInlineArrayElementSize()) - 10 + + // Create a large child array which will NOT be inlined in parent container. + childArray2 := interpreter.NewArrayValue( + inter, + interpreter.EmptyLocationRange, + interpreter.VariableSizedStaticType{ + Type: interpreter.PrimitiveStaticTypeAnyStruct, + }, + owner, + interpreter.NewUnmeteredStringValue(strings.Repeat("b", size)), + interpreter.NewUnmeteredStringValue(strings.Repeat("c", size)), + ) + + // Create an array with childArray1 and childArray2. + array := interpreter.NewArrayValue( + inter, + interpreter.EmptyLocationRange, + interpreter.VariableSizedStaticType{ + Type: interpreter.PrimitiveStaticTypeAnyStruct, + }, + owner, + childArray1, // inlined + childArray2, // not inlined + ) + + // DeepRemove removes all elements (childArray1 and childArray2) recursively in array. + array.DeepRemove(inter, true) + + // As noted earlier in comments at the top of this test: + // storage.CheckHealth() should be called after array.DeepRemove(), not in the middle of array.DeepRemove(). + // This happens when: + // - array.DeepRemove() calls childArray1 and childArray2 DeepRemove() + // - DeepRemove() calls maybeValidateAtreeValue() + // - maybeValidateAtreeValue() calls CheckHealth() +} + +// This test is a reproducer for "slab was not reachable from leaves" false alarm. +// https://github.com/onflow/cadence/pull/2882#issuecomment-1796381227 +// In this test, storage.CheckHealth() should be called after DictionaryValue.Transfer() +// with remove flag, not in the middle of DictionaryValue.Transfer(). +func TestCheckStorageHealthInMiddleOfTransferAndRemove(t *testing.T) { + r := newRandomValueGenerator() + t.Logf("seed: %d", r.seed) + + storage := newUnmeteredInMemoryStorage() + inter, err := interpreter.NewInterpreter( + &interpreter.Program{ + Program: ast.NewProgram(nil, []ast.Declaration{}), + Elaboration: sema.NewElaboration(nil), + }, + utils.TestLocation, + &interpreter.Config{ + Storage: storage, + ImportLocationHandler: func(inter *interpreter.Interpreter, location common.Location) interpreter.Import { + return interpreter.VirtualImport{ + Elaboration: inter.Program.Elaboration, + } + }, + AtreeStorageValidationEnabled: true, + AtreeValueValidationEnabled: true, + }, + ) + require.NoError(t, err) + + // Create large array value with zero address which will not be inlined. + gchildArray := interpreter.NewArrayValue( + inter, + interpreter.EmptyLocationRange, + interpreter.VariableSizedStaticType{ + Type: interpreter.PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + interpreter.NewUnmeteredStringValue(strings.Repeat("b", int(atree.MaxInlineArrayElementSize())-10)), + interpreter.NewUnmeteredStringValue(strings.Repeat("c", int(atree.MaxInlineArrayElementSize())-10)), + ) + + // Create small composite value with zero address which will be inlined. + identifier := "test" + + location := common.AddressLocation{ + Address: common.ZeroAddress, + Name: identifier, + } + + compositeType := &sema.CompositeType{ + Location: location, + Identifier: identifier, + Kind: common.CompositeKindStructure, + } + + fields := []interpreter.CompositeField{ + interpreter.NewUnmeteredCompositeField("a", interpreter.NewUnmeteredUInt64Value(0)), + interpreter.NewUnmeteredCompositeField("b", interpreter.NewUnmeteredUInt64Value(1)), + interpreter.NewUnmeteredCompositeField("c", interpreter.NewUnmeteredUInt64Value(2)), + } + + compositeType.Members = &sema.StringMemberOrderedMap{} + for _, field := range fields { + compositeType.Members.Set( + field.Name, + sema.NewUnmeteredPublicConstantFieldMember( + compositeType, + field.Name, + sema.AnyStructType, + "", + ), + ) + } + + // Add the type to the elaboration, to short-circuit the type-lookup. + inter.Program.Elaboration.SetCompositeType( + compositeType.ID(), + compositeType, + ) + + gchildComposite := interpreter.NewCompositeValue( + inter, + interpreter.EmptyLocationRange, + location, + identifier, + common.CompositeKindStructure, + fields, + common.ZeroAddress, + ) + + // Create large dictionary with zero address with 2 data slabs containing: + // - SomeValue(SlabID) as first physical element in the first data slab + // - inlined CompositeValue as last physical element in the second data slab + + numberOfValues := 10 + firstElementIndex := 7 // index of first physical element in the first data slab + lastElementIndex := 8 // index of last physical element in the last data slab + keyValues := make([]interpreter.Value, numberOfValues*2) + for i := 0; i < numberOfValues; i++ { + key := interpreter.NewUnmeteredUInt64Value(uint64(i)) + + var value interpreter.Value + switch i { + case firstElementIndex: + value = interpreter.NewUnmeteredSomeValueNonCopying(gchildArray) + + case lastElementIndex: + value = gchildComposite + + default: + // Other values are inlined random strings. + const size = 235 + value = interpreter.NewUnmeteredStringValue(r.randomUTF8StringOfSize(size)) + } + + keyValues[i*2] = key + keyValues[i*2+1] = value + } + + childMap := interpreter.NewDictionaryValueWithAddress( + inter, + interpreter.EmptyLocationRange, + interpreter.DictionaryStaticType{ + KeyType: interpreter.PrimitiveStaticTypeAnyStruct, + ValueType: interpreter.PrimitiveStaticTypeAnyStruct, + }, + common.ZeroAddress, + keyValues..., + ) + + // Create dictionary with non-zero address containing child dictionary. + owner := common.Address{'A'} + m := interpreter.NewDictionaryValueWithAddress( + inter, + interpreter.EmptyLocationRange, + interpreter.DictionaryStaticType{ + KeyType: interpreter.PrimitiveStaticTypeAnyStruct, + ValueType: interpreter.PrimitiveStaticTypeAnyStruct, + }, + owner, + interpreter.NewUnmeteredUInt64Value(0), + childMap, + ) + + inter.ValidateAtreeValue(m) + + require.NoError(t, storage.CheckHealth()) +}