public archive sdk

pkg/node/helper.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
@@ -24,8 +26,11 @@ import (
 	"github.com/ava-labs/avalanche-cli/pkg/utils"
 	"github.com/ava-labs/avalanche-cli/pkg/ux"
 	"github.com/ava-labs/avalanche-cli/pkg/vm"
+	"github.com/ava-labs/avalanche-cli/sdk/network"
+	"github.com/ava-labs/avalanche-cli/sdk/publicarchive"
 	"github.com/ava-labs/avalanche-network-runner/rpcpb"
 	"github.com/ava-labs/avalanchego/api/info"
+	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/utils/set"
 )
 
@@ -555,3 +560,75 @@ func GetNodeData(endpoint string) (
 		"0x" + hex.EncodeToString(proofOfPossession.ProofOfPossession[:]),
 		nil
 }
+
+func DownloadPublicArchive(
+	clusterNetwork models.Network,
+	rootDir string,
+	nodeNames []string,
+) error {
+	// only fuji is supported for now
+	if clusterNetwork.Kind != models.Fuji {
+		return fmt.Errorf("unsupported network: %s", clusterNetwork.Name())
+	}
+	network := network.FujiNetwork()
+	ux.Logger.Info("downloading public archive for network %s", clusterNetwork.Name())
+	publicArcDownloader, err := publicarchive.NewDownloader(network, logging.NewLogger("public-archive-downloader", logging.NewWrappedCore(logging.Off, os.Stdout, logging.JSON.ConsoleEncoder()))) // off as we run inside of the spinner
+	if err != nil {
+		return fmt.Errorf("failed to create public archive downloader for network %s: %w", clusterNetwork.Name(), err)
+	}
+
+	if err := publicArcDownloader.Download(); err != nil {
+		return fmt.Errorf("failed to download public archive: %w", err)
+	}
+	defer publicArcDownloader.CleanUp()
+	if path, err := publicArcDownloader.GetFilePath(); err != nil {
+		return fmt.Errorf("failed to get downloaded file path: %w", err)
+	} else {
+		ux.Logger.Info("public archive downloaded to %s", path)
+	}
+
+	wg := sync.WaitGroup{}
+	mu := sync.Mutex{}
+	var firstErr error
+
+	for _, nodeName := range nodeNames {
+		target := filepath.Join(rootDir, nodeName, "db")
+		ux.Logger.Info("unpacking public archive to %s", target)
+
+		// Skip if target already exists
+		if _, err := os.Stat(target); err == nil {
+			ux.Logger.Info("data folder already exists at %s. Skipping...", target)
+			continue
+		}
+		wg.Add(1)
+		go func(target string) {
+			defer wg.Done()
+
+			if err := publicArcDownloader.UnpackTo(target); err != nil {
+				// Capture the first error encountered
+				mu.Lock()
+				if firstErr == nil {
+					firstErr = fmt.Errorf("failed to unpack public archive: %w", err)
+					_ = cleanUpClusterNodeData(rootDir, nodeNames)
+				}
+				mu.Unlock()
+			}
+		}(target)
+	}
+	wg.Wait()
+
+	if firstErr != nil {
+		return firstErr
+	}
+	ux.Logger.PrintToUser("Public archive unpacked to: %s", rootDir)
+	return nil
+}
+
+func cleanUpClusterNodeData(rootDir string, nodesNames []string) error {
+	for _, nodeName := range nodesNames {
+		if err := os.RemoveAll(filepath.Join(rootDir, nodeName)); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/node/local.go

@@ -427,6 +427,14 @@ func StartLocalNode(
 		ux.Logger.PrintToUser("Starting local avalanchego node using root: %s ...", rootDir)
 		spinSession := ux.NewUserSpinner()
 		spinner := spinSession.SpinToUser("Booting Network. Wait until healthy...")
+		// preseed nodes data from public archive. ignore errors
+		nodeNames := []string{}
+		for i := 1; i <= int(numNodes); i++ {
+			nodeNames = append(nodeNames, fmt.Sprintf("node%d", i))
+		}
+		err := DownloadPublicArchive(network, rootDir, nodeNames)
+		ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err)
+
 		if _, err := cli.Start(ctx, avalancheGoBinPath, anrOpts...); err != nil {
 			ux.SpinFailWithError(spinner, "", err)
 			_ = DestroyLocalNode(app, clusterName)
@@ -490,6 +498,9 @@ func UpsizeLocalNode(
 		nodeConfig = map[string]interface{}{}
 	}
 	nodeConfig[config.NetworkAllowPrivateIPsKey] = true
+	if network.Kind == models.Fuji {
+		nodeConfig[config.IndexEnabledKey] = false // disable index for Fuji
+	}
 	nodeConfigBytes, err := json.Marshal(nodeConfig)
 	if err != nil {
 		return "", err
@@ -586,6 +597,8 @@ func UpsizeLocalNode(
 
 	spinSession := ux.NewUserSpinner()
 	spinner := spinSession.SpinToUser("Creating new node with name %s on local machine", newNodeName)
+	err = DownloadPublicArchive(network, rootDir, []string{newNodeName})
+	ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err)
 	// add new local node
 	if _, err := cli.AddNode(ctx, newNodeName, avalancheGoBinPath, anrOpts...); err != nil {
 		ux.SpinFailWithError(spinner, "", err)

sdk/constants/constants.go

@@ -10,5 +10,6 @@ const (
 	APIRequestLargeTimeout = 2 * time.Minute
 
 	// node
-	WriteReadUserOnlyPerms = 0o600
+	UserOnlyWriteReadPerms     = 0o600
+	UserOnlyWriteReadExecPerms = 0o700
 )

sdk/key/soft_key.go

@@ -280,7 +280,7 @@ func (m *SoftKey) PrivKeyHex() string {
 
 // Saves the private key to disk with hex encoding.
 func (m *SoftKey) Save(p string) error {
-	return os.WriteFile(p, []byte(m.PrivKeyHex()), constants.WriteReadUserOnlyPerms)
+	return os.WriteFile(p, []byte(m.PrivKeyHex()), constants.UserOnlyWriteReadPerms)
 }
 
 func (m *SoftKey) P(networkHRP string) (string, error) {

sdk/publicarchive/README.md

@@ -0,0 +1,55 @@
+# Public Archive Downloader SDK
+
+This Go package provides a utility to download and extract tar archives from public URLs. It's tailored for downloading Avalanche network archives but can be adapted for other use cases.
+
+
+## Features
+
+* Downloads files from predefined URLs.
+* Tracks download progress and logs status updates.
+* Safely unpacks .tar archives to a target directory.
+* Includes security checks to prevent path traversal and manage large files.
+
+## Usage example
+
+```
+// Copyright (C) 2024, Ava Labs, Inc. All rights reserved
+// See the file LICENSE for licensing terms.
+
+```
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ava-labs/avalanche-cli/sdk/network"
+	"github.com/ava-labs/avalanchego/utils/constants"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/your-repo-name/publicarchive"
+)
+
+func main() {
+	// Initialize the downloader
+	downloader, err := publicarchive.NewDownloader(network.FujiNetwork(), logging.Debug)
+	if err != nil {
+		fmt.Printf("Failed to create downloader: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Start downloading
+	if err := downloader.Download(); err != nil {
+		fmt.Printf("Download failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Specify the target directory for unpacking
+	targetDir := "./extracted_files"
+	if err := downloader.UnpackTo(targetDir); err != nil {
+		fmt.Printf("Failed to unpack archive: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Files successfully unpacked to %s\n", targetDir)
+}
+```