diff --git a/.github/workflows/wormchain-icts.yml b/.github/workflows/wormchain-icts.yml index d8bb38b526..867c729875 100644 --- a/.github/workflows/wormchain-icts.yml +++ b/.github/workflows/wormchain-icts.yml @@ -14,13 +14,45 @@ permissions: env: GO_VERSION: 1.21 + TAR_PATH: /tmp/wormchain-docker-image.tar + IMAGE_NAME: wormchain-docker-image concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + build-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: wormchain/interchaintest/go.sum + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and export + uses: docker/build-push-action@v5 + with: + context: . + file: wormchain/Dockerfile.ict + tags: wormchain:local + outputs: type=docker,dest=${{ env.TAR_PATH }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ env.IMAGE_NAME }} + path: ${{ env.TAR_PATH }} + e2e-tests: + needs: build-docker runs-on: ubuntu-latest strategy: matrix: @@ -30,6 +62,7 @@ jobs: - "ictest-upgrade" - "ictest-wormchain" - "ictest-ibc-receiver" + - "ictest-validator-hotswap" - "ictest-cw-wormhole" fail-fast: false @@ -43,6 +76,17 @@ jobs: - name: checkout chain uses: actions/checkout@v4 + - name: Download Tarball Artifact + uses: actions/download-artifact@v3 + with: + name: ${{ env.IMAGE_NAME }} + path: /tmp + + - name: Load Docker Image + run: | + docker image load -i ${{ env.TAR_PATH }} + docker image ls -a + - name: Run Test id: run_test continue-on-error: true diff --git a/wormchain/Dockerfile.ict b/wormchain/Dockerfile.ict new file mode 100644 index 0000000000..f186b286cc --- /dev/null +++ b/wormchain/Dockerfile.ict @@ -0,0 +1,40 @@ +FROM golang:1.22.5@sha256:86a3c48a61915a8c62c0e1d7594730399caa3feb73655dfe96c7bc17710e96cf AS builder + +WORKDIR /app + +# Install dependencies +RUN apt update && \ + apt-get install -y \ + build-essential \ + ca-certificates \ + curl + +# Enable faster module downloading. +ENV GOPROXY https://proxy.golang.org + +COPY ./wormchain/go.mod . +COPY ./wormchain/go.sum . +COPY ./sdk /sdk +RUN go mod download + +COPY ./wormchain . + +RUN make build/wormchaind + +FROM golang:1.22.5@sha256:86a3c48a61915a8c62c0e1d7594730399caa3feb73655dfe96c7bc17710e96cf + +WORKDIR /home/heighliner + +COPY --from=builder /app/build/wormchaind /usr/bin + +# copy over c bindings (libwasmvm.x86_64.so, etc) +COPY --from=builder /go/pkg/mod/github.com/!cosm!wasm/wasmvm@v1.1.1/internal/api/* /usr/lib + +EXPOSE 26657 +EXPOSE 26656 +EXPOSE 6060 +EXPOSE 9090 +EXPOSE 1317 +EXPOSE 4500 + +ENTRYPOINT [ "wormchaind" ] \ No newline at end of file diff --git a/wormchain/Makefile b/wormchain/Makefile index ac8819f70e..7fbdb3a6be 100644 --- a/wormchain/Makefile +++ b/wormchain/Makefile @@ -83,6 +83,10 @@ clean: ## INTERCHAINTESTS ## ##################### +# Generate Wormchain Image +local-image: build/wormchaind + docker build -t wormchain:local -f Dockerfile.ict .. + # Individual Tests ($$ is interpreted as $) rm-testcache: go clean -testcache @@ -108,4 +112,7 @@ ictest-ibc-receiver: rm-testcache ictest-cw-wormhole: rm-testcache cd interchaintest && go test -race -v -run ^TestCWWormhole ./... -.PHONY: ictest-cancel-upgrade ictest-malformed-payload ictest-upgrade-failure ictest-upgrade ictest-wormchain ictest-ibc-receiver ictest-cw-wormhole \ No newline at end of file +ictest-validator-hotswap: rm-testcache + cd interchaintest && go test -race -v -run ^TestValidatorHotswap$$ ./... + +.PHONY: ictest-cancel-upgrade ictest-malformed-payload ictest-upgrade-failure ictest-upgrade ictest-wormchain ictest-ibc-receiver ictest-cw-wormhole ictest-validator-hotswap diff --git a/wormchain/interchaintest/cw_wormhole_test.go b/wormchain/interchaintest/cw_wormhole_test.go index 9ce1054964..bd59557838 100644 --- a/wormchain/interchaintest/cw_wormhole_test.go +++ b/wormchain/interchaintest/cw_wormhole_test.go @@ -28,7 +28,7 @@ func createSingleNodeCluster(t *testing.T, wormchainVersion string, guardians gu numFullNodes := 0 wormchainConfig.Images[0].Version = wormchainVersion - wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, true) + wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, numWormchainVals, true) cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ { diff --git a/wormchain/interchaintest/hot_swap_test.go b/wormchain/interchaintest/hot_swap_test.go new file mode 100644 index 0000000000..864ee21efd --- /dev/null +++ b/wormchain/interchaintest/hot_swap_test.go @@ -0,0 +1,231 @@ +package ictest + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/strangelove-ventures/interchaintest/v4" + "github.com/strangelove-ventures/interchaintest/v4/chain/cosmos" + "github.com/strangelove-ventures/interchaintest/v4/ibc" + "github.com/strangelove-ventures/interchaintest/v4/testreporter" + "github.com/strangelove-ventures/interchaintest/v4/testutil" + "github.com/stretchr/testify/require" + "github.com/wormhole-foundation/wormchain/interchaintest/guardians" + "github.com/wormhole-foundation/wormchain/interchaintest/helpers" + "go.uber.org/zap/zaptest" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/ethereum/go-ethereum/crypto" + wormholetypes "github.com/wormhole-foundation/wormchain/x/wormhole/types" + wormholesdk "github.com/wormhole-foundation/wormhole/sdk" +) + +func SetupHotSwapChain(t *testing.T, wormchainVersion string, guardians guardians.ValSet, numVals int) ibc.Chain { + wormchainConfig.Images[0].Version = wormchainVersion + + if wormchainVersion == "local" { + wormchainConfig.Images[0].Repository = "wormchain" + } + + // Create chain factory with wormchain + wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, numVals, true) + + numFullNodes := 0 + cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ + { + ChainName: "wormchain", + ChainConfig: wormchainConfig, + NumValidators: &numVals, + NumFullNodes: &numFullNodes, + }, + }) + + // Get chains from the chain factory + chains, err := cf.Chains(t.Name()) + require.NoError(t, err) + + return chains[0] +} + +type ValidatorInfo struct { + Validator *cosmos.ChainNode + Bech32Addr string + AccAddr sdk.AccAddress +} + +type QueryAllGuardianValidatorResponse struct { + GuardianValidators []GuardianValidator `json:"guardianValidator"` +} + +type QueryGetGuardianValidatorResponse struct { + GuardianValidator GuardianValidator `json:"guardianValidator"` +} + +func TestValidatorHotswap(t *testing.T) { + // Base setup + numGuardians := 2 + numVals := 3 + guardians := guardians.CreateValSet(t, numGuardians) + chain := SetupHotSwapChain(t, "local", *guardians, numVals) + + ic := interchaintest.NewInterchain().AddChain(chain) + ctx := context.Background() + rep := testreporter.NewNopReporter() + eRep := rep.RelayerExecReporter(t) + client, network := interchaintest.DockerSetup(t) + + err := ic.Build(ctx, eRep, interchaintest.InterchainBuildOptions{ + TestName: t.Name(), + Client: client, + NetworkID: network, + SkipPathCreation: true, + }) + require.NoError(t, err) + + t.Cleanup(func() { + _ = ic.Close() + }) + + wormchain := chain.(*cosmos.CosmosChain) + + // ============================ + + // Query active guardian validators (returns both keys & sdk acc address) + res, _, err := wormchain.Validators[0].ExecQuery(ctx, "wormhole", "list-guardian-validator") + require.NoError(t, err) + + // Validate response + var guardianValidators QueryAllGuardianValidatorResponse + err = json.Unmarshal(res, &guardianValidators) + require.NoError(t, err) + require.Equal(t, numGuardians, len(guardianValidators.GuardianValidators)) + + // ============================ + + // NOTE: + // + // wormchain.Validators & the guardan query do not guarantee order, so we need to map the validators to match the order + // of the guardian set reference. + + // First guardian key refs - will swap from using first validator to last validator, then back again + firstGuardianKey := guardianValidators.GuardianValidators[0].GuardianKey + firstGuardianPrivKey := guardians.Vals[0].Priv + if !bytes.Equal(firstGuardianKey, guardians.Vals[0].Addr) { + firstGuardianPrivKey = guardians.Vals[1].Priv + } + + // Guardian validatore sdk addresses + firstGuardianValAddr := sdk.AccAddress(guardianValidators.GuardianValidators[0].ValidatorAddr) + secondGuardianValAddr := sdk.AccAddress(guardianValidators.GuardianValidators[1].ValidatorAddr) + + // Map validators to guardian set order + var validators [3]ValidatorInfo + for _, val := range wormchain.Validators { + valBech32Addr, err := val.AccountKeyBech32(ctx, "validator") + require.NoError(t, err) + + valInfo := ValidatorInfo{ + Validator: val, + Bech32Addr: valBech32Addr, + AccAddr: helpers.MustAccAddressFromBech32(valBech32Addr, "wormhole"), + } + + if strings.Contains(valInfo.AccAddr.String(), firstGuardianValAddr.String()) { + validators[0] = valInfo + } else if strings.Contains(valInfo.AccAddr.String(), secondGuardianValAddr.String()) { + validators[1] = valInfo + } else { + validators[2] = valInfo + } + } + + // Ensure all validators are mapped + require.NotNil(t, validators[0]) + require.NotNil(t, validators[1]) + require.NotNil(t, validators[2]) + + // References to first & last validator + firstVal := validators[0] + newVal := validators[2] + + // ============================ + + // Ensure chain can produce blocks with the last validator shut down, + // as it is not in the active set + newVal.Validator.StopContainer(ctx) + err = testutil.WaitForBlocks(ctx, 10, wormchain) + require.NoError(t, err) + newVal.Validator.StartContainer(ctx) + + // ============================ + + // Query the first guardian's validator + guardianKey := hex.EncodeToString(firstGuardianKey) + res, _, err = newVal.Validator.ExecQuery(ctx, "wormhole", "show-guardian-validator", guardianKey) + require.NoError(t, err) + + // Ensure the first guardian's validator is set to the first validator + var valResponse wormholetypes.QueryGetGuardianValidatorResponse + err = json.Unmarshal(res, &valResponse) + require.NoError(t, err) + require.Equal(t, firstGuardianKey, valResponse.GuardianValidator.GuardianKey) + require.Equal(t, firstVal.AccAddr.Bytes(), valResponse.GuardianValidator.ValidatorAddr) + + // ============================ + + // Use first validator to allow list the last validator (as it is not in active set) + _, err = firstVal.Validator.ExecTx(ctx, "validator", "wormhole", "create-allowed-address", newVal.Bech32Addr, "newVal") + require.NoError(t, err) + + // Migrate first guardian to use last validator + addrHash := crypto.Keccak256Hash(wormholesdk.SignedWormchainAddressPrefix, newVal.AccAddr) + sig, err := crypto.Sign(addrHash[:], firstGuardianPrivKey) + require.NoErrorf(t, err, "failed to sign wormchain address: %v", err) + _, err = newVal.Validator.ExecTx(ctx, "validator", "wormhole", "register-account-as-guardian", hex.EncodeToString(sig)) + require.NoError(t, err) + + // Query the first guardian's validator + res, _, err = newVal.Validator.ExecQuery(ctx, "wormhole", "show-guardian-validator", guardianKey) + require.NoError(t, err) + + // Ensure the first guardian's validator is set to the last validator + err = json.Unmarshal(res, &valResponse) + require.NoError(t, err) + require.Equal(t, firstGuardianKey, valResponse.GuardianValidator.GuardianKey) + require.Equal(t, newVal.AccAddr.Bytes(), valResponse.GuardianValidator.ValidatorAddr) + + // Wait 10 blocks to ensure blocks are being produced + err = testutil.WaitForBlocks(ctx, 10, wormchain) + require.NoError(t, err) + + // ============================ + + // Use last validator to allow list the first validator (as it is not in active set *anymore) + _, err = newVal.Validator.ExecTx(ctx, "validator", "wormhole", "create-allowed-address", firstVal.Bech32Addr, "firstVal") + require.NoError(t, err) + + // Migrate first guardian back to use first validator + addrHash = crypto.Keccak256Hash(wormholesdk.SignedWormchainAddressPrefix, firstVal.AccAddr) + sig, err = crypto.Sign(addrHash[:], firstGuardianPrivKey) + require.NoErrorf(t, err, "failed to sign wormchain address: %v", err) + _, err = firstVal.Validator.ExecTx(ctx, "validator", "wormhole", "register-account-as-guardian", hex.EncodeToString(sig)) + require.NoError(t, err) + + // Query the first guardian's validator + res, _, err = firstVal.Validator.ExecQuery(ctx, "wormhole", "show-guardian-validator", guardianKey) + require.NoError(t, err) + + // Ensure the first guardian's validator is set to the first validator + err = json.Unmarshal(res, &valResponse) + require.NoError(t, err) + require.Equal(t, firstGuardianKey, valResponse.GuardianValidator.GuardianKey) + require.Equal(t, firstVal.AccAddr.Bytes(), valResponse.GuardianValidator.ValidatorAddr) + + // Wait 10 blocks to ensure blocks are being produced + err = testutil.WaitForBlocks(ctx, 10, wormchain) + require.NoError(t, err) +} diff --git a/wormchain/interchaintest/ibc_receiver_test.go b/wormchain/interchaintest/ibc_receiver_test.go index cf4380c80b..a26fceafdf 100644 --- a/wormchain/interchaintest/ibc_receiver_test.go +++ b/wormchain/interchaintest/ibc_receiver_test.go @@ -34,7 +34,7 @@ func createChains(t *testing.T, wormchainVersion string, guardians guardians.Val wormchainConfig.Images[0].Version = wormchainVersion // Create chain factory with wormchain - wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, false) + wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, len(guardians.Vals), false) cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ { diff --git a/wormchain/interchaintest/setup.go b/wormchain/interchaintest/setup.go index e418f4de17..c545c55916 100644 --- a/wormchain/interchaintest/setup.go +++ b/wormchain/interchaintest/setup.go @@ -69,8 +69,12 @@ func CreateChains(t *testing.T, wormchainVersion string, guardians guardians.Val numWormchainVals := len(guardians.Vals) wormchainConfig.Images[0].Version = wormchainVersion + if wormchainVersion == "local" { + wormchainConfig.Images[0].Repository = "wormchain" + } + // Create chain factory with wormchain - wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, false) + wormchainConfig.ModifyGenesis = ModifyGenesis(votingPeriod, maxDepositPeriod, guardians, len(guardians.Vals), false) cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ { @@ -167,9 +171,8 @@ func BuildInterchain(t *testing.T, chains []ibc.Chain) (context.Context, ibc.Rel // * Set Guardian Set List using new val set // * Set Guardian Validator List using new val set // * Allow list the faucet address -func ModifyGenesis(votingPeriod string, maxDepositPeriod string, guardians guardians.ValSet, skipRelayers bool) func(ibc.ChainConfig, []byte) ([]byte, error) { +func ModifyGenesis(votingPeriod string, maxDepositPeriod string, guardians guardians.ValSet, numVals int, skipRelayers bool) func(ibc.ChainConfig, []byte) ([]byte, error) { return func(chainConfig ibc.ChainConfig, genbz []byte) ([]byte, error) { - numVals := len(guardians.Vals) g := make(map[string]interface{}) if err := json.Unmarshal(genbz, &g); err != nil { return nil, fmt.Errorf("failed to unmarshal genesis file: %w", err) @@ -218,7 +221,7 @@ func ModifyGenesis(votingPeriod string, maxDepositPeriod string, guardians guard Keys: [][]byte{}, } guardianValidators := []GuardianValidator{} - for i := 0; i < numVals; i++ { + for i := 0; i < len(guardians.Vals); i++ { guardianSet.Keys = append(guardianSet.Keys, guardians.Vals[i].Addr) guardianValidators = append(guardianValidators, GuardianValidator{ GuardianKey: guardians.Vals[i].Addr, diff --git a/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian.go b/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian.go index 4cef14435b..02657a0898 100644 --- a/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian.go +++ b/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian.go @@ -14,8 +14,6 @@ import ( // This function is used to onboard Wormhole Guardians as Validators on Wormchain. // It creates a 1:1 association between a Guardian addresss and a Wormchain validator address. -// There is also a special case -- when the size of the Guardian set is 1, the Guardian is allowed to "hot-swap" their validator address in the mapping. -// We include the special case to make it easier to shuffle things in testnets and local devnets. // 1. Guardian signs their validator address -- SIGNATURE=$(guardiand admin sign-wormchain-address ) // 2. Guardian submits $SIGNATURE to Wormchain via this handler, using their new validator address as the signer of the Wormchain tx. func (k msgServer) RegisterAccountAsGuardian(goCtx context.Context, msg *types.MsgRegisterAccountAsGuardian) (*types.MsgRegisterAccountAsGuardianResponse, error) { @@ -25,6 +23,7 @@ func (k msgServer) RegisterAccountAsGuardian(goCtx context.Context, msg *types.M if err != nil { return nil, err } + // recover guardian key from signature signerHash := crypto.Keccak256Hash(wormholesdk.SignedWormchainAddressPrefix, signer) guardianKey, err := crypto.Ecrecover(signerHash.Bytes(), msg.Signature) @@ -50,16 +49,14 @@ func (k msgServer) RegisterAccountAsGuardian(goCtx context.Context, msg *types.M return nil, types.ErrGuardianSetNotFound } - consensusGuardianSetIndex, consensusIndexFound := k.GetConsensusGuardianSetIndex(ctx) + // With the change to allow hot swapping on validator sets > 1 validator, this check is no + // longer useful. However, it is necessary. This check ensures the gas usage of the transaction + // matches the previous implementation as this is included in a non-consensus breaking change. + _, consensusIndexFound := k.GetConsensusGuardianSetIndex(ctx) if !consensusIndexFound { return nil, types.ErrConsensusSetUndefined } - // If the size of the guardian set is 1, allow hot-swapping the validator address. - if consensusIndexFound && latestGuardianSetIndex == consensusGuardianSetIndex.Index && len(latestGuardianSet.Keys) > 1 { - return nil, types.ErrConsensusSetNotUpdatable - } - if !latestGuardianSet.ContainsKey(guardianKeyAddr) { return nil, types.ErrGuardianNotFound } diff --git a/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian_test.go b/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian_test.go index 8011491b9d..7a5fc7f8fb 100644 --- a/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian_test.go +++ b/wormchain/x/wormhole/keeper/msg_server_register_account_as_guardian_test.go @@ -53,8 +53,8 @@ func TestRegisterAccountAsGuardianHotSwap(t *testing.T) { assert.Equal(t, newValAddr.Bytes(), newGuardian.ValidatorAddr) } -// disallow hot-swapping validator addresses when guardian set size is >1 -func TestRegisterAccountAsGuardianBlockHotSwap(t *testing.T) { +// test hot swapping with validator size > 1 +func TestRegisterAccountAsGuardianHotSwapMultipleValidators(t *testing.T) { // setup -- create guardian set of size 2 k, ctx := keepertest.WormholeKeeper(t) guardians, privateKeys := createNGuardianValidator(k, ctx, 2) @@ -64,8 +64,6 @@ func TestRegisterAccountAsGuardianBlockHotSwap(t *testing.T) { ChainId: uint32(vaa.ChainIDWormchain), GuardianSetExpiration: 86400, }) - newValAddr_bz := [20]byte{} - newValAddr := sdk.AccAddress(newValAddr_bz[:]) set := createNewGuardianSet(k, ctx, guardians) k.SetConsensusGuardianSetIndex(ctx, types.ConsensusGuardianSetIndex{Index: set.Index}) @@ -74,15 +72,50 @@ func TestRegisterAccountAsGuardianBlockHotSwap(t *testing.T) { context := sdk.WrapSDKContext(ctx) msgServer := keeper.NewMsgServerImpl(*k) + // store old val addr for later + + oldValAddr_bz := [20]byte{} + copy(oldValAddr_bz[:], guardians[0].ValidatorAddr) + oldValAddr := sdk.AccAddress(oldValAddr_bz[:]) + + // hot swap to new val addr + + newValAddr_bz := [20]byte{} + newValAddr := sdk.AccAddress(newValAddr_bz[:]) + // sign the new validator address as the new validator address addrHash := crypto.Keccak256Hash(wormholesdk.SignedWormchainAddressPrefix, newValAddr) sig, err := crypto.Sign(addrHash[:], privateKeys[0]) require.NoErrorf(t, err, "failed to sign wormchain address: %v", err) - // assert that we are unable to associate the guardian address with a new validator address when the set size is >1 + // assert we can hot swap when validators > 1 _, err = msgServer.RegisterAccountAsGuardian(context, &types.MsgRegisterAccountAsGuardian{ Signer: newValAddr.String(), Signature: sig, }) - assert.Error(t, types.ErrConsensusSetNotUpdatable, err) + assert.NoError(t, err) + + // assert that the guardian validator has the new validator address + newGuardian, newGuardianFound := k.GetGuardianValidator(ctx, guardians[0].GuardianKey) + require.Truef(t, newGuardianFound, "expected guardian not found in the keeper store") + assert.Equal(t, newValAddr.Bytes(), newGuardian.ValidatorAddr) + + // -- hot swap back to old val addr -- + + // sign the old validator address as the new validator address + addrHash = crypto.Keccak256Hash(wormholesdk.SignedWormchainAddressPrefix, oldValAddr) + sig, err = crypto.Sign(addrHash[:], privateKeys[0]) + require.NoErrorf(t, err, "failed to sign wormchain address: %v", err) + + // assert we can hot swap back to the old validator address + _, err = msgServer.RegisterAccountAsGuardian(context, &types.MsgRegisterAccountAsGuardian{ + Signer: oldValAddr.String(), + Signature: sig, + }) + assert.NoError(t, err) + + // assert that the guardian validator has the old validator address + oldGuardian, oldGuardianFound := k.GetGuardianValidator(ctx, guardians[0].GuardianKey) + require.Truef(t, oldGuardianFound, "expected guardian not found in the keeper store") + assert.Equal(t, oldValAddr.Bytes(), oldGuardian.ValidatorAddr) }