test: tss migration e2e test

Makefile

@@ -87,7 +87,7 @@ build-testnet-ubuntu: go.sum
 		docker rm temp-container
 
 install: go.sum
-		@echo "--> Installing zetacored & zetaclientd"
+		@echo "--> Installing zetacored ,zetaclientd and zetaclientd-supervisor"
 		@go install -mod=readonly $(BUILD_FLAGS) ./cmd/zetacored
 		@go install -mod=readonly $(BUILD_FLAGS) ./cmd/zetaclientd
 		@go install -mod=readonly $(BUILD_FLAGS) ./cmd/zetaclientd-supervisor
@@ -254,6 +254,11 @@ start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose --profile stress -f docker-compose.yml up -d
 
+start-tss-migration-test: zetanode
+	@echo "--> Starting migration test"
+	export E2E_ARGS="--test-migration" && \
+	cd contrib/localnet/ && $(DOCKER) compose up -d
+
 ###############################################################################
 ###                         Upgrade Tests              						###
 ###############################################################################

changelog.md

@@ -76,6 +76,7 @@
 * [2349](https://github.com/zeta-chain/node/pull/2349) - add TestBitcoinDepositRefund and WithdrawBitcoinMultipleTimes E2E tests
 * [2368](https://github.com/zeta-chain/node/pull/2368) - eliminate panic usage across testing suite
 * [2369](https://github.com/zeta-chain/node/pull/2369) - fix random cross-chain swap failure caused by using tiny UTXO
+* [2440](https://github.com/zeta-chain/node/pull/2440) - Add e2e test for Tss migration
 
 
 ### Fixes

cmd/zetaclientd-supervisor/lib.go

@@ -12,6 +12,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/client/grpc/tmservice"
@@ -20,6 +21,7 @@ import (
 	"github.com/rs/zerolog"
 	"google.golang.org/grpc"
 
+	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 	"github.com/zeta-chain/zetacore/zetaclient/config"
 )
 
@@ -66,6 +68,7 @@ type zetaclientdSupervisor struct {
 	upgradesDir        string
 	upgradePlanName    string
 	enableAutoDownload bool
+	restartChan        chan os.Signal
 }
 
 func newZetaclientdSupervisor(
@@ -81,19 +84,24 @@ func newZetaclientdSupervisor(
 	if err != nil {
 		return nil, fmt.Errorf("grpc dial: %w", err)
 	}
-
+	// these signals will result in the supervisor process only restarting zetaclientd
+	restartChan := make(chan os.Signal, 1)
 	return &zetaclientdSupervisor{
 		zetacoredConn:      conn,
 		logger:             logger,
 		reloadSignals:      make(chan bool, 1),
 		upgradesDir:        defaultUpgradesDir,
 		enableAutoDownload: enableAutoDownload,
+		restartChan:        restartChan,
 	}, nil
 }
 
 func (s *zetaclientdSupervisor) Start(ctx context.Context) {
 	go s.watchForVersionChanges(ctx)
 	go s.handleCoreUpgradePlan(ctx)
+	go s.handleNewKeygen(ctx)
+	go s.handleNewTssKeyGeneration(ctx)
+	go s.handleTssUpdate(ctx)
 }
 
 func (s *zetaclientdSupervisor) WaitForReloadSignal(ctx context.Context) {
@@ -169,6 +177,122 @@ func (s *zetaclientdSupervisor) watchForVersionChanges(ctx context.Context) {
 	}
 }
 
+func (s *zetaclientdSupervisor) handleTssUpdate(ctx context.Context) {
+	maxRetries := 11
+	retryInterval := 5 * time.Second
+
+	for i := 0; i < maxRetries; i++ {
+		client := observertypes.NewQueryClient(s.zetacoredConn)
+		tss, err := client.TSS(ctx, &observertypes.QueryGetTSSRequest{})
+		if err != nil {
+			s.logger.Warn().Err(err).Msg("unable to get original tss")
+			time.Sleep(retryInterval)
+			continue
+		}
+		i = 0
+		for {
+			select {
+			case <-time.After(time.Second):
+			case <-ctx.Done():
+				return
+			}
+			tssNew, err := client.TSS(ctx, &observertypes.QueryGetTSSRequest{})
+			if err != nil {
+				s.logger.Warn().Err(err).Msg("unable to get tss")
+				continue
+			}
+
+			if tssNew.TSS.TssPubkey == tss.TSS.TssPubkey {
+				continue
+			}
+
+			tss = tssNew
+			s.logger.Info().
+				Msgf("tss address is updated from %s to %s", tss.TSS.TssPubkey, tssNew.TSS.TssPubkey)
+			time.Sleep(6 * time.Second)
+			s.logger.Info().Msg("restarting zetaclientd to update tss address")
+			s.restartChan <- syscall.SIGHUP
+		}
+	}
+	s.logger.Warn().Msg("handleTssUpdate exiting without success")
+}
+
+func (s *zetaclientdSupervisor) handleNewTssKeyGeneration(ctx context.Context) {
+	maxRetries := 11
+	retryInterval := 5 * time.Second
+
+	for i := 0; i < maxRetries; i++ {
+		client := observertypes.NewQueryClient(s.zetacoredConn)
+		alltss, err := client.TssHistory(ctx, &observertypes.QueryTssHistoryRequest{})
+		if err != nil {
+			s.logger.Warn().Err(err).Msg("unable to get tss original history")
+			time.Sleep(retryInterval)
+			continue
+		}
+		i = 0
+		tssLenCurrent := len(alltss.TssList)
+		for {
+			select {
+			case <-time.After(time.Second):
+			case <-ctx.Done():
+				return
+			}
+			tssListNew, err := client.TssHistory(ctx, &observertypes.QueryTssHistoryRequest{})
+			if err != nil {
+				s.logger.Warn().Err(err).Msg("unable to get tss new history")
+				continue
+			}
+			tssLenUpdated := len(tssListNew.TssList)
+
+			if tssLenUpdated == tssLenCurrent {
+				continue
+			}
+			if tssLenUpdated < tssLenCurrent {
+				tssLenCurrent = len(tssListNew.TssList)
+				continue
+			}
+
+			tssLenCurrent = tssLenUpdated
+			s.logger.Info().Msgf("tss list updated from %d to %d", tssLenCurrent, tssLenUpdated)
+			time.Sleep(5 * time.Second)
+			s.logger.Info().Msg("restarting zetaclientd to update tss list")
+			s.restartChan <- syscall.SIGHUP
+		}
+	}
+	s.logger.Warn().Msg("handleNewTssKeyGeneration exiting without success")
+}
+
+func (s *zetaclientdSupervisor) handleNewKeygen(ctx context.Context) {
+	client := observertypes.NewQueryClient(s.zetacoredConn)
+	prevKeygenBlock := int64(0)
+	for {
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return
+		}
+		resp, err := client.Keygen(ctx, &observertypes.QueryGetKeygenRequest{})
+		if err != nil {
+			s.logger.Warn().Err(err).Msg("unable to get keygen")
+			continue
+		}
+		if resp.Keygen == nil {
+			s.logger.Warn().Err(err).Msg("keygen is nil")
+			continue
+		}
+
+		if resp.Keygen.Status != observertypes.KeygenStatus_PendingKeygen {
+			continue
+		}
+		keygenBlock := resp.Keygen.BlockNumber
+		if prevKeygenBlock == keygenBlock {
+			continue
+		}
+		prevKeygenBlock = keygenBlock
+		s.logger.Info().Msgf("got new keygen at block %d", keygenBlock)
+		s.restartChan <- syscall.SIGHUP
+	}
+}
 func (s *zetaclientdSupervisor) handleCoreUpgradePlan(ctx context.Context) {
 	client := upgradetypes.NewQueryClient(s.zetacoredConn)
 

cmd/zetaclientd-supervisor/main.go

@@ -36,10 +36,6 @@ func main() {
 	shutdownChan := make(chan os.Signal, 1)
 	signal.Notify(shutdownChan, syscall.SIGINT, syscall.SIGTERM)
 
-	// these signals will result in the supervisor process only restarting zetaclientd
-	restartChan := make(chan os.Signal, 1)
-	signal.Notify(restartChan, syscall.SIGHUP)
-
 	hotkeyPassword, tssPassword, err := promptPasswords()
 	if err != nil {
 		logger.Error().Err(err).Msg("unable to get passwords")
@@ -53,6 +49,7 @@ func main() {
 		os.Exit(1)
 	}
 	supervisor.Start(ctx)
+	signal.Notify(supervisor.restartChan, syscall.SIGHUP)
 
 	shouldRestart := true
 	for shouldRestart {
@@ -82,7 +79,7 @@ func main() {
 				select {
 				case <-ctx.Done():
 					return nil
-				case sig := <-restartChan:
+				case sig := <-supervisor.restartChan:
 					logger.Info().Msgf("got signal %d, sending SIGINT to zetaclientd", sig)
 				case sig := <-shutdownChan:
 					logger.Info().Msgf("got signal %d, shutting down", sig)