Acp77 remote etna devnet (#2233)

* etna devnet constants

* add custom bootrap conf to node create

* Update cmd/nodecmd/create.go

Co-authored-by: Michael Kaplan <55204436+michaelkaplan13@users.noreply.github.com>
Signed-off-by: arturrez <56270896+arturrez@users.noreply.github.com>

* add unittests for ip:port pair verificatoion

* update etna bootstrap info

* add  AvalancheGoConfigOptions struct

* etnadevnet network model

* write data to files

* err handling

* support etna devnet

* fix address input

* allow public access to rpc endpoint for etna devnet

* add ux to report public access to user

* read bls info and nodeid for the cluster

* added 0x to string, added log

* fix issue with http_host reverted after sync

* resolve conflict

* lint

* after merge fixes

---------

Signed-off-by: arturrez <56270896+arturrez@users.noreply.github.com>
Co-authored-by: Michael Kaplan <55204436+michaelkaplan13@users.noreply.github.com>

cmd/blockchaincmd/deploy.go

@@ -219,7 +219,7 @@ func getChainsInSubnet(blockchainName string) ([]string, error) {
 }
 
 func checkSubnetEVMDefaultAddressNotInAlloc(network models.Network, chain string) error {
-	if network.Kind != models.Local && network.Kind != models.Devnet && os.Getenv(constants.SimulatePublicNetwork) == "" {
+	if network.Kind != models.Local && network.Kind != models.Devnet && network.Kind != models.EtnaDevnet && os.Getenv(constants.SimulatePublicNetwork) == "" {
 		genesis, err := app.LoadEvmGenesis(chain)
 		if err != nil {
 			return err
@@ -410,6 +410,26 @@ func deployBlockchain(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if sidecar.Sovereign && bootstrapValidatorsJSONFilePath == "" {
+		// TODO: add check for local cluster from another PR
+		if len(bootstrapValidators) == 0 && globalNetworkFlags.ClusterName != "" {
+			// get bootstrap validators from cluster
+			changeOwnerAddr, err := getKeyForChangeOwner("", "", network)
+			if err != nil {
+				return err
+			}
+			bootstrapValidators, err = getClusterBootstrapValidators(globalNetworkFlags.ClusterName, changeOwnerAddr)
+			if err != nil {
+				return err
+			}
+		} else {
+			bootstrapValidators, err = promptBootstrapValidators(network)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	ux.Logger.PrintToUser("Deploying %s to %s", chains, network.Name())
 
 	if network.Kind == models.Local {

cmd/blockchaincmd/prompt_genesis_input.go

@@ -3,6 +3,7 @@
 package blockchaincmd
 
 import (
+	"encoding/hex"
 	"fmt"
 
 	"github.com/ava-labs/avalanche-cli/pkg/application"
@@ -120,6 +121,31 @@ func generateNewNodeAndBLS() (string, string, string, error) {
 	return nodeID.String(), publicKey, pop, nil
 }
 
+func getClusterBootstrapValidators(clusterName string, changeOwnerAddr string) ([]models.SubnetValidator, error) {
+	clusterConf, err := app.GetClusterConfig(clusterName)
+	if err != nil {
+		return nil, err
+	}
+	subnetValidators := []models.SubnetValidator{}
+	hostIDs := utils.Filter(clusterConf.GetCloudIDs(), clusterConf.IsAvalancheGoHost)
+	for _, h := range hostIDs {
+		id, pub, pop, err := utils.GetNodeParams(app.GetNodeInstanceDirPath(h))
+		if err != nil {
+			return nil, err
+		}
+		ux.Logger.Info("Bootstrap validator info for Host: %s | Node ID: %s | Public Key: %s | Proof of Possession: %s", h, id, hex.EncodeToString(pub), hex.EncodeToString(pop))
+		subnetValidators = append(subnetValidators, models.SubnetValidator{
+			NodeID:               id.String(),
+			Weight:               constants.BootstrapValidatorWeight,
+			Balance:              constants.BootstrapValidatorBalance,
+			BLSPublicKey:         fmt.Sprintf("%s%s", "0x", hex.EncodeToString(pub)),
+			BLSProofOfPossession: fmt.Sprintf("%s%s", "0x", hex.EncodeToString(pop)),
+			ChangeOwnerAddr:      changeOwnerAddr,
+		})
+	}
+	return subnetValidators, nil
+}
+
 func promptBootstrapValidators(network models.Network) ([]models.SubnetValidator, error) {
 	var subnetValidators []models.SubnetValidator
 	numBootstrapValidators, err := app.Prompt.CaptureInt(

cmd/nodecmd/create.go

@@ -45,7 +45,7 @@ const (
 )
 
 var (
-	createSupportedNetworkOptions         = []networkoptions.NetworkOption{networkoptions.Fuji, networkoptions.Devnet}
+	createSupportedNetworkOptions         = []networkoptions.NetworkOption{networkoptions.Fuji, networkoptions.Devnet, networkoptions.EtnaDevnet}
 	globalNetworkFlags                    networkoptions.NetworkFlags
 	useAWS                                bool
 	useGCP                                bool
@@ -125,6 +125,10 @@ will apply to all nodes in the cluster`,
 	cmd.Flags().IntVar(&volumeSize, "aws-volume-size", constants.CloudServerStorageSize, "AWS volume size in GB")
 	cmd.Flags().BoolVar(&replaceKeyPair, "auto-replace-keypair", false, "automatically replaces key pair to access node if previous key pair is not found")
 	cmd.Flags().BoolVar(&publicHTTPPortAccess, "public-http-port", false, "allow public access to avalanchego HTTP port")
+	cmd.Flags().StringArrayVar(&bootstrapIDs, "bootstrap-ids", []string{}, "nodeIDs of bootstrap nodes")
+	cmd.Flags().StringArrayVar(&bootstrapIPs, "bootstrap-ips", []string{}, "IP:port pairs of bootstrap nodes")
+	cmd.Flags().StringVar(&genesisPath, "genesis", "", "path to genesis file")
+	cmd.Flags().StringVar(&upgradePath, "upgrade", "", "path to upgrade file")
 	return cmd
 }
 
@@ -202,8 +206,27 @@ func preCreateChecks(clusterName string) error {
 		return err
 	}
 	if clusterConfig.Local {
-		return notImplementedForLocal("addDashboard")
+		return notImplementedForLocal("create")
+	} // bootsrap checks
+	if globalNetworkFlags.UseEtnaDevnet && (len(bootstrapIDs) != 0 || len(bootstrapIPs) != 0 || genesisPath != "" || upgradePath != "") {
+		return fmt.Errorf("etna devnet uses predefined bootsrap configuration")
 	}
+	if len((bootstrapIDs)) != len(bootstrapIPs) {
+		return fmt.Errorf("number of bootstrap ids and ip:port pairs must be equal")
+	}
+	if genesisPath != "" && !utils.FileExists(genesisPath) {
+		return fmt.Errorf("genesis file %s does not exist", genesisPath)
+	}
+	if upgradePath != "" && !utils.FileExists(upgradePath) {
+		return fmt.Errorf("upgrade file %s does not exist", upgradePath)
+	}
+	// check ip:port pairs
+	for _, ipPortPair := range bootstrapIPs {
+		if ok := utils.IsValidIPPort(ipPortPair); !ok {
+			return fmt.Errorf("invalid ip:port pair %s", ipPortPair)
+		}
+	}
+
 	return nil
 }
 
@@ -245,9 +268,6 @@ func stringToAWSVolumeType(input string) types.VolumeType {
 
 func createNodes(cmd *cobra.Command, args []string) error {
 	clusterName := args[0]
-	if err := preCreateChecks(clusterName); err != nil {
-		return err
-	}
 	network, err := networkoptions.GetNetworkFromCmdLineFlags(
 		app,
 		"",
@@ -257,9 +277,44 @@ func createNodes(cmd *cobra.Command, args []string) error {
 		createSupportedNetworkOptions,
 		"",
 	)
-	if err != nil {
+	if err := preCreateChecks(clusterName); err != nil {
 		return err
 	}
+	if network.Kind == models.EtnaDevnet {
+		publicHTTPPortAccess = true // public http port access for etna devnet api for PoAManagerDeployment
+		bootstrapIDs = constants.EtnaDevnetBootstrapNodeIDs
+		bootstrapIPs = constants.EtnaDevnetBootstrapIPs
+
+		// create genesis and upgrade files
+		genesisTmpFile, err := os.CreateTemp("", "genesis")
+		if err != nil {
+			return err
+		}
+		if _, err := genesisTmpFile.Write(constants.EtnaDevnetGenesisData); err != nil {
+			return err
+		}
+		if err := genesisTmpFile.Close(); err != nil {
+			return err
+		}
+		genesisPath = genesisTmpFile.Name()
+
+		upgradeTmpFile, err := os.CreateTemp("", "upgrade")
+		if err != nil {
+			return err
+		}
+		if _, err := upgradeTmpFile.Write(constants.EtnaDevnetUpgradeData); err != nil {
+			return err
+		}
+		if err := upgradeTmpFile.Close(); err != nil {
+			return err
+		}
+		upgradePath = upgradeTmpFile.Name()
+
+		defer func() {
+			_ = os.Remove(genesisTmpFile.Name())
+			_ = os.Remove(upgradeTmpFile.Name())
+		}()
+	}
 	network = models.NewNetworkFromCluster(network, clusterName)
 	globalNetworkFlags.UseDevnet = network.Kind == models.Devnet // set globalNetworkFlags.UseDevnet to true if network is devnet for further use
 	avaGoVersionSetting := node.AvalancheGoVersionSettings{
@@ -725,7 +780,15 @@ func createNodes(cmd *cobra.Command, args []string) error {
 			spinner = spinSession.SpinToUser(utils.ScriptLog(host.NodeID, "Setup AvalancheGo"))
 			// check if host is a API host
 			publicAccessToHTTPPort := slices.Contains(cloudConfigMap.GetAllAPIInstanceIDs(), host.GetCloudID()) || publicHTTPPortAccess
-			if err := docker.ComposeSSHSetupNode(host, network, avalancheGoVersion, addMonitoring, publicAccessToHTTPPort); err != nil {
+			if err := docker.ComposeSSHSetupNode(host,
+				network,
+				avalancheGoVersion,
+				bootstrapIDs,
+				bootstrapIPs,
+				genesisPath,
+				upgradePath,
+				addMonitoring,
+				publicAccessToHTTPPort); err != nil {
 				nodeResults.AddResult(host.NodeID, nil, err)
 				ux.SpinFailWithError(spinner, "", err)
 				return

cmd/nodecmd/local.go

@@ -76,7 +76,6 @@ status by running avalanche node status local
 	cmd.Flags().StringArrayVar(&bootstrapIPs, "bootstrap-ip", []string{}, "IP:port pairs of bootstrap nodes")
 	cmd.Flags().StringVar(&genesisPath, "genesis", "", "path to genesis file")
 	cmd.Flags().StringVar(&upgradePath, "upgrade", "", "path to upgrade file")
-	cmd.Flags().BoolVar(&useEtnaDevnet, "etna-devnet", false, "use Etna devnet. Prepopulated with Etna DevNet bootstrap configuration along with genesis and upgrade files")
 	cmd.Flags().StringVar(&stakingTLSKeyPath, "staking-tls-key-path", "", "path to provided staking tls key for node")
 	cmd.Flags().StringVar(&stakingCertKeyPath, "staking-cert-key-path", "", "path to provided staking cert key for node")
 	cmd.Flags().StringVar(&stakingSignerKeyPath, "staking-signer-key-path", "", "path to provided staking signer key for node")

cmd/nodecmd/upgrade.go

@@ -59,7 +59,6 @@ func upgrade(_ *cobra.Command, args []string) error {
 	if clusterConfig.Local {
 		return notImplementedForLocal("upgrade")
 	}
-	network := clusterConfig.Network
 	hosts, err := ansible.GetInventoryFromAnsibleInventoryFile(app.GetAnsibleInventoryDirPath(clusterName))
 	if err != nil {
 		return err
@@ -73,9 +72,7 @@ func upgrade(_ *cobra.Command, args []string) error {
 	for host, upgradeInfo := range toUpgradeNodesMap {
 		if upgradeInfo.AvalancheGoVersion != "" {
 			spinner := spinSession.SpinToUser(utils.ScriptLog(host.NodeID, fmt.Sprintf("Upgrading avalanchego to version %s...", upgradeInfo.AvalancheGoVersion)))
-			// check if host is API host
-			publicAccessToHTTPPort := clusterConfig.IsAPIHost(host.GetCloudID()) || clusterConfig.HTTPAccess == constants.PublicAccess
-			if err := upgradeAvalancheGo(host, network, upgradeInfo.AvalancheGoVersion, publicAccessToHTTPPort); err != nil {
+			if err := upgradeAvalancheGo(host, upgradeInfo.AvalancheGoVersion); err != nil {
 				ux.SpinFailWithError(spinner, "", err)
 				return err
 			}
@@ -221,11 +218,9 @@ func checkIfKeyIsStandardVMName(vmName string) bool {
 
 func upgradeAvalancheGo(
 	host *models.Host,
-	network models.Network,
 	avaGoVersionToUpdateTo string,
-	publicAccessToHTTPPort bool,
 ) error {
-	if err := ssh.RunSSHUpgradeAvalanchego(host, network, avaGoVersionToUpdateTo, publicAccessToHTTPPort); err != nil {
+	if err := ssh.RunSSHUpgradeAvalanchego(host, avaGoVersionToUpdateTo); err != nil {
 		return err
 	}
 	return nil

pkg/docker/compose.go

@@ -19,7 +19,7 @@ import (
 	"github.com/ava-labs/avalanche-cli/pkg/ux"
 )
 
-type dockerComposeInputs struct {
+type DockerComposeInputs struct {
 	WithMonitoring     bool
 	WithAvalanchego    bool
 	AvalanchegoVersion string
@@ -31,7 +31,7 @@ type dockerComposeInputs struct {
 //go:embed templates/*.docker-compose.yml
 var composeTemplate embed.FS
 
-func renderComposeFile(composePath string, composeDesc string, templateVars dockerComposeInputs) ([]byte, error) {
+func renderComposeFile(composePath string, composeDesc string, templateVars DockerComposeInputs) ([]byte, error) {
 	compose, err := composeTemplate.ReadFile(composePath)
 	if err != nil {
 		return nil, err
@@ -206,7 +206,7 @@ func ComposeOverSSH(
 	host *models.Host,
 	timeout time.Duration,
 	composePath string,
-	composeVars dockerComposeInputs,
+	composeVars DockerComposeInputs,
 ) error {
 	remoteComposeFile := utils.GetRemoteComposeFile()
 	startTime := time.Now()

pkg/docker/config.go

@@ -5,18 +5,40 @@ package docker
 
 import (
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/ava-labs/avalanche-cli/pkg/constants"
 	"github.com/ava-labs/avalanche-cli/pkg/models"
 	"github.com/ava-labs/avalanche-cli/pkg/remoteconfig"
 	"github.com/ava-labs/avalanche-cli/pkg/utils"
 )
 
-func prepareAvalanchegoConfig(host *models.Host, network models.Network, publicAccess bool) (string, string, error) {
+type AvalancheGoConfigOptions struct {
+	BootstrapIPs      []string
+	BootstrapIDs      []string
+	GenesisPath       string
+	UpgradePath       string
+	AllowPublicAccess bool
+}
+
+func prepareAvalanchegoConfig(
+	host *models.Host,
+	network models.Network,
+	avalancheGoConfig AvalancheGoConfigOptions,
+) (string, string, error) {
 	avagoConf := remoteconfig.PrepareAvalancheConfig(host.IP, network.NetworkIDFlagValue(), nil)
-	if publicAccess || utils.IsE2E() {
+	if avalancheGoConfig.AllowPublicAccess || utils.IsE2E() {
 		avagoConf.HTTPHost = "0.0.0.0"
 	}
+	avagoConf.BootstrapIPs = strings.Join(avalancheGoConfig.BootstrapIPs, ",")
+	avagoConf.BootstrapIDs = strings.Join(avalancheGoConfig.BootstrapIDs, ",")
+	if avalancheGoConfig.GenesisPath != "" {
+		avagoConf.GenesisPath = filepath.Join(constants.DockerNodeConfigPath, constants.GenesisFileName)
+	}
+	if avalancheGoConfig.UpgradePath != "" {
+		avagoConf.UpgradePath = filepath.Join(constants.DockerNodeConfigPath, constants.UpgradeFileName)
+	}
 	nodeConf, err := remoteconfig.RenderAvalancheNodeConfig(avagoConf)
 	if err != nil {
 		return "", "", err

pkg/docker/ssh.go

@@ -25,7 +25,17 @@ func ValidateComposeFile(host *models.Host, composeFile string, timeout time.Dur
 }
 
 // ComposeSSHSetupNode sets up an AvalancheGo node and dependencies on a remote host over SSH.
-func ComposeSSHSetupNode(host *models.Host, network models.Network, avalancheGoVersion string, withMonitoring bool, publicAccessToHTTPPort bool) error {
+func ComposeSSHSetupNode(
+	host *models.Host,
+	network models.Network,
+	avalancheGoVersion string,
+	avalanchegoBootstrapIDs []string,
+	avalanchegoBootstrapIPs []string,
+	avalanchegoGenesisFilePath string,
+	avalanchegoUpgradeFilePath string,
+	withMonitoring bool,
+	publicAccessToHTTPPort bool,
+) error {
 	startTime := time.Now()
 	folderStructure := remoteconfig.RemoteFoldersToCreateAvalanchego()
 	for _, dir := range folderStructure {
@@ -41,7 +51,17 @@ func ComposeSSHSetupNode(host *models.Host, network models.Network, avalancheGoV
 		return err
 	}
 	ux.Logger.Info("AvalancheGo Docker image %s ready on %s[%s] after %s", avagoDockerImage, host.NodeID, host.IP, time.Since(startTime))
-	nodeConfFile, cChainConfFile, err := prepareAvalanchegoConfig(host, network, publicAccessToHTTPPort)
+	nodeConfFile, cChainConfFile, err := prepareAvalanchegoConfig(
+		host,
+		network,
+		AvalancheGoConfigOptions{
+			BootstrapIDs:      avalanchegoBootstrapIDs,
+			BootstrapIPs:      avalanchegoBootstrapIPs,
+			GenesisPath:       avalanchegoGenesisFilePath,
+			UpgradePath:       avalanchegoUpgradeFilePath,
+			AllowPublicAccess: publicAccessToHTTPPort,
+		},
+	)
 	if err != nil {
 		return err
 	}
@@ -60,12 +80,22 @@ func ComposeSSHSetupNode(host *models.Host, network models.Network, avalancheGoV
 	if err := host.Upload(cChainConfFile, remoteconfig.GetRemoteAvalancheCChainConfig(), constants.SSHFileOpsTimeout); err != nil {
 		return err
 	}
+	if avalanchegoGenesisFilePath != "" {
+		if err := host.Upload(avalanchegoGenesisFilePath, remoteconfig.GetRemoteAvalancheGenesis(), constants.SSHFileOpsTimeout); err != nil {
+			return err
+		}
+	}
+	if avalanchegoUpgradeFilePath != "" {
+		if err := host.Upload(avalanchegoUpgradeFilePath, remoteconfig.GetRemoteAvalancheUpgrade(), constants.SSHFileOpsTimeout); err != nil {
+			return err
+		}
+	}
 	ux.Logger.Info("AvalancheGo configs uploaded to %s[%s] after %s", host.NodeID, host.IP, time.Since(startTime))
 	return ComposeOverSSH("Compose Node",
 		host,
 		constants.SSHScriptTimeout,
 		"templates/avalanchego.docker-compose.yml",
-		dockerComposeInputs{
+		DockerComposeInputs{
 			AvalanchegoVersion: avalancheGoVersion,
 			WithMonitoring:     withMonitoring,
 			WithAvalanchego:    true,
@@ -80,7 +110,7 @@ func ComposeSSHSetupLoadTest(host *models.Host) error {
 		host,
 		constants.SSHScriptTimeout,
 		"templates/avalanchego.docker-compose.yml",
-		dockerComposeInputs{
+		DockerComposeInputs{
 			WithMonitoring:  true,
 			WithAvalanchego: false,
 		})
@@ -133,13 +163,13 @@ func ComposeSSHSetupMonitoring(host *models.Host) error {
 		host,
 		constants.SSHScriptTimeout,
 		"templates/monitoring.docker-compose.yml",
-		dockerComposeInputs{})
+		DockerComposeInputs{})
 }
 
 func ComposeSSHSetupAWMRelayer(host *models.Host) error {
 	return ComposeOverSSH("Setup AWM Relayer",
 		host,
 		constants.SSHScriptTimeout,
 		"templates/awmrelayer.docker-compose.yml",
-		dockerComposeInputs{})
+		DockerComposeInputs{})
 }