Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve logging and e2e script #99

Merged
merged 1 commit into from
Feb 24, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 70 additions & 49 deletions scripts/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ CONDUIT_VERSION="$default_version"
FALLBACK_BRANCH="dev"
BUILD_SCRIPT="./scripts/build.sh"

DEBUG=0
CLEAN=0

OUT_BINARY=""

while [[ $# -gt 0 ]]; do
Expand All @@ -25,6 +28,14 @@ while [[ $# -gt 0 ]]; do
CONDUIT_VERSION="$2"
shift 2
;;
--clean)
CLEAN=1
shift
;;
--debug)
DEBUG=1
shift
;;
*)
echo "Unknown argument: $1"
exit 1
Expand All @@ -42,60 +53,64 @@ function download_and_extract() {
local repo_url=$3
local dest_dir="$dependency_dir/$repo_name"

echo "Attemting to download $repo_name"
echo "Processing dependency: $repo_name"

OUT_BINARY=""

# Remove existing directory to ensure fresh download
if [ -d "$dest_dir" ]; then
echo "Removing existing $repo_name directory..."
rm -rf "$dest_dir"
fi

mkdir -p "$dest_dir"
release_version=""
if [ "$version" = "latest" ]; then
release_version=$(curl -s "https://api.github.com/repos/chain4travel/$repo_name/releases/latest" | grep -Po '"tag_name": "\K[^"]*' || echo "")
fi

if [ -z "$release_version" ] ; then
if [ "$version" = "latest" ]; then
branch=$FALLBACK_BRANCH
else
branch=$version
fi
if [ $CLEAN -eq 1 ] ; then
echo "Removing existing $repo_name directory..."
rm -rf "$dest_dir"
fi

echo "WARN: Unable to get the released version of $repo_name! Fallback to clone and build of the branch '$branch'."
release_version=""
if [ "$version" = "latest" ]; then
release_version=$(curl -s "https://api.github.com/repos/chain4travel/$repo_name/releases/latest" | grep -Po '"tag_name": "\K[^"]*' || echo "")
fi

if git ls-remote --heads --tags "$repo_url" | grep -q "$branch"; then
git clone --depth 1 --branch "$branch" "$repo_url" "$dest_dir"
elif git ls-remote "$repo_url" | grep -q "$branch"; then
git clone --depth 1 "$repo_url" "$dest_dir"
cd "$dest_dir"
git checkout "$branch"
else
echo "Version/tag/commit '$branch' not found for $repo_name, aborting."
exit 1
fi

cd "${ORIG_DIR}/$dest_dir"
if [ ! -f $BUILD_SCRIPT ] ; then
echo "CRIT: No build script found at '$BUILD_SCRIPT' in cloned repository. Abort."
exit 1
fi
$BUILD_SCRIPT
cd "$ORIG_DIR"
else
local url="https://github.com/chain4travel/$repo_name/releases/download/$release_version/${repo_name}-linux-amd64-${release_version}.tar.gz"

echo "Downloading $repo_name version $release_version..."
if curl --output /dev/null --silent --head --fail "$url"; then
curl -s -L "$url" -o "$dest_dir/${repo_name}.tar.gz"
tar -xzf "$dest_dir/${repo_name}.tar.gz" -C "$dest_dir"
rm "$dest_dir/${repo_name}.tar.gz"
else
echo "CRIT: Unable to download the release '$release_version' of $repo_name."
exit 1
if [ -d "$dest_dir" ] ; then
echo "Directory $dest_dir already exists. Skipping download and build."
else
mkdir -p "$dest_dir"
if [ -z "$release_version" ] ; then
if [ "$version" = "latest" ]; then
branch=$FALLBACK_BRANCH
else
branch=$version
fi

echo "WARN: Unable to get the released version of $repo_name! Fallback to clone and build of the branch '$branch'."

if git ls-remote --heads --tags "$repo_url" | grep -q "$branch"; then
git clone --depth 1 --branch "$branch" "$repo_url" "$dest_dir"
elif git ls-remote "$repo_url" | grep -q "$branch"; then
git clone --depth 1 "$repo_url" "$dest_dir"
cd "$dest_dir"
git checkout "$branch"
else
echo "Version/tag/commit '$branch' not found for $repo_name, aborting."
exit 1
fi

cd "${ORIG_DIR}/$dest_dir"
if [ ! -f $BUILD_SCRIPT ] ; then
echo "CRIT: No build script found at '$BUILD_SCRIPT' in cloned repository. Abort."
exit 1
fi
$BUILD_SCRIPT
cd "$ORIG_DIR"
else
local url="https://github.com/chain4travel/$repo_name/releases/download/$release_version/${repo_name}-linux-amd64-${release_version}.tar.gz"

echo "Downloading $repo_name version $release_version..."
if curl --output /dev/null --silent --head --fail "$url"; then
curl -s -L "$url" -o "$dest_dir/${repo_name}.tar.gz"
tar -xzf "$dest_dir/${repo_name}.tar.gz" -C "$dest_dir"
rm "$dest_dir/${repo_name}.tar.gz"
else
echo "CRIT: Unable to download the release '$release_version' of $repo_name."
exit 1
fi
fi
fi

Expand Down Expand Up @@ -165,10 +180,16 @@ CMB_DB_MIGRATIONS_PATH="$(realpath "${CMB_DB_MIGRATIONS_PATH}")"

echo "Running e2e tests..."

ADD_PARAM=()
if [ $DEBUG -eq 1 ] ; then
ADD_PARAM+=("-debug")
fi

./$E2E_BIN_OUT \
-test.v \
-node="${CAMINOGO_BIN_PATH}" \
-matrix="${MATRIX_BIN_PATH}" \
-partner-plugin="${PARTNER_PLUGIN_BIN_PATH}" \
-cmb="${CMB_BIN_PATH}" \
-migration="${CMB_DB_MIGRATIONS_PATH}"
-migration="${CMB_DB_MIGRATIONS_PATH}" \
"${ADD_PARAM[@]}"
14 changes: 7 additions & 7 deletions tests/e2e/blockchain/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func StartNewNetwork(
nodeBinPath string,
validatorsCount int,
) (*Network, chan error, error) {
logger.Infof("Starting blockchain network (%d validators)...", validatorsCount)
logger.Debugf("Starting blockchain network (%d validators)...", validatorsCount)

var err error
httpPorts := make([]int, validatorsCount)
Expand Down Expand Up @@ -180,19 +180,19 @@ func StartNewNetwork(

n.Client = n.nodes[0].client

logger.Info("Preparing EVM admin...")
logger.Debug("Preparing EVM admin...")

if err := n.Client.prepareAdmin(ctx); err != nil {
return n, nil, fmt.Errorf("failed to prepare EVM admin: %w", err)
}

logger.Info("Preparing CMB contracts...")
logger.Debug("Preparing CMB contracts...")

if err := n.Client.prepareCMBContracts(ctx); err != nil {
return n, nil, fmt.Errorf("failed to prepare CMB contracts: %w", err)
}

logger.Info("Blockchain network started")
logger.Debug("Blockchain network started")

return n, combineErrChannels(errChans...), nil
}
Expand Down Expand Up @@ -258,7 +258,7 @@ func (n *Network) startNewNode(
bootstrapIPsArg string,
nodeIndex int,
) (*Node, chan error, error) {
n.logger.Infof("Starting blockchain node %d (http port %d)...", nodeIndex, httpPort)
n.logger.Debugf("Starting blockchain node %d (http port %d)...", nodeIndex, httpPort)

if err := os.RemoveAll(nodeDir); err != nil {
return nil, nil, fmt.Errorf("failed to remove blockchain node tmp dir: %w", err)
Expand Down Expand Up @@ -310,7 +310,7 @@ func (n *Network) startNewNode(
}
node.client = client

n.logger.Infof("Blockchain node %d (pid %d) started", nodeIndex, node.pid)
n.logger.Debugf("Blockchain node %d (pid %d) started", nodeIndex, node.pid)

errChan := make(chan error)
go func() {
Expand All @@ -333,7 +333,7 @@ func (n *Network) Stop(ctx context.Context) error {
return fmt.Errorf("failed to stop node: %w", err)
}
}
n.logger.Info("Blockchain network stopped")
n.logger.Debug("Blockchain network stopped")
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/blockchain/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func (n *Node) Stop(ctx context.Context) error {
if err := process.StopProcess(ctx, n.pid); err != nil {
return fmt.Errorf("failed to stop node process with pid %d: %w", n.pid, err)
}
n.logger.Infof("Blockchain node (pid %d) stopped", n.pid)
n.logger.Debugf("Blockchain node (pid %d) stopped", n.pid)
if err := n.logfile.Close(); err != nil {
return fmt.Errorf("failed to close node logfile: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/bot/bot.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func (b *Bot) Stop(ctx context.Context) error {
if err := process.StopProcess(ctx, b.pid); err != nil {
return fmt.Errorf("failed to stop cmb process with pid %d: %w", b.pid, err)
}
b.logger.Infof("Bot (pid %d) stopped", b.pid)
b.logger.Debugf("Bot (pid %d) stopped", b.pid)
if err := b.logfile.Close(); err != nil {
return fmt.Errorf("failed to close partner plugin logfile: %w", err)
}
Expand Down
4 changes: 2 additions & 2 deletions tests/e2e/bot/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,10 +212,10 @@ func (f *Factory) CreateBot(
return bot, nil, fmt.Errorf("failed to await bot readiness: %w", err)
}
} else {
f.logger.Warnf("bot (pid %d) started without RPC server: no readiness check", cmd.Process.Pid)
f.logger.Debugf("bot (pid %d) started without RPC server: no readiness check", cmd.Process.Pid)
}

f.logger.Infof("bot (pid %d) started", cmd.Process.Pid)
f.logger.Debugf("bot (pid %d) started", cmd.Process.Pid)

f.bots = append(f.bots, bot)

Expand Down
8 changes: 4 additions & 4 deletions tests/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
flagKeyPartnerPluginBinPath = "partner-plugin"
flagKeyCMBBinPath = "cmb"
flagKeyMigrationsDir = "migration"
flagKeyDebug = "debug"
)

var (
Expand All @@ -40,21 +41,19 @@ var (
flagTestsDataDir string
flagExistingNetworkNodeURI string
flagExistingNetworkAdminKey string
flagDebug bool
)

func init() {
flag.StringVar(&flagNodeBinPath, flagKeyNodeBinPath, "", "Path to node binary.")
flag.StringVar(&flagExistingNetworkNodeURI, "existing-network-node-uri", "", "URI of existing network node.")
flag.StringVar(&flagExistingNetworkAdminKey, "existing-network-admin-key", "", "Admin key of existing network.")

flag.StringVar(&flagMatrixBinPath, flagKeyMatrixBinPath, "", "Path to matrix binary.")

flag.StringVar(&flagPartnerPluginBinPath, flagKeyPartnerPluginBinPath, "", "Path to partner plugin binary.")

flag.StringVar(&flagCMBBinPath, flagKeyCMBBinPath, "", "Path to CMB binary.")
flag.StringVar(&flagMigrationsDir, flagKeyMigrationsDir, "", "Path to migration files dir.")

flag.StringVar(&flagTestsDataDir, "tests-data-dir", "/tmp/cmb-e2e", "Path to dir with temp tests data.")
flag.BoolVar(&flagDebug, flagKeyDebug, false, "Debug mode")
}

func TestE2E(t *testing.T) {
Expand Down Expand Up @@ -103,6 +102,7 @@ func TestE2E(t *testing.T) {
flagTestsDataDir,
flagExistingNetworkNodeURI,
existingNetworkAdminKey,
flagDebug,
)
require.NoError(t, err)

Expand Down
6 changes: 3 additions & 3 deletions tests/e2e/matrix/conduit.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func StartNewMatrixServer(
networkFeeKey *ecdsa.PrivateKey,
networkClient *blockchain.Client,
) (*MatrixServer, chan error, error) {
logger.Info("Starting matrix server...")
logger.Debug("Starting matrix server...")

matrixDir := path.Join(dataDir, "matrix")
if err := os.RemoveAll(matrixDir); err != nil {
Expand Down Expand Up @@ -119,7 +119,7 @@ func StartNewMatrixServer(
return nil, nil, fmt.Errorf("failed to add bot to CM account: %w", err)
}

logger.Infof("Matrix server (pid %d) started", cmd.Process.Pid)
logger.Debugf("Matrix server (pid %d) started", cmd.Process.Pid)

errChan := make(chan error)
go func() {
Expand Down Expand Up @@ -160,7 +160,7 @@ func (m *MatrixServer) Stop(ctx context.Context) error {
if err := m.logfile.Close(); err != nil {
return fmt.Errorf("failed to close matrix server logfile: %w", err)
}
m.logger.Infof("Matrix server (pid %d) stopped", m.pid)
m.logger.Debugf("Matrix server (pid %d) stopped", m.pid)
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/partner_plugin/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (f *Factory) CreatePartnerPlugin(ctx context.Context) (*PartnerPlugin, chan
return pp, nil, fmt.Errorf("failed to wait for partner-plugin to be ready: %w", err)
}

f.logger.Infof("Partner-plugin (pid %d) started", cmd.Process.Pid)
f.logger.Debugf("Partner-plugin (pid %d) started", cmd.Process.Pid)

f.partnerPlugins = append(f.partnerPlugins, pp)

Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/partner_plugin/partner_plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (pp *PartnerPlugin) Stop(ctx context.Context) error {
if err := process.StopProcess(ctx, pp.pid); err != nil {
return fmt.Errorf("failed to stop partner plugin process with pid %d: %w", pp.pid, err)
}
pp.logger.Infof("Partner plugin (pid %d) stopped", pp.pid)
pp.logger.Debugf("Partner plugin (pid %d) stopped", pp.pid)
if err := pp.logfile.Close(); err != nil {
return fmt.Errorf("failed to close partner plugin logfile: %w", err)
}
Expand Down
13 changes: 10 additions & 3 deletions tests/e2e/tests/suite.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,15 @@ func NewSuite(
testsDataDir string,
existingNetworkNodeURI string,
existingNetworkAdminKey *secp256k1.PrivateKey,
debug bool,
) (*Suite, error) {
logger, err := zap.NewDevelopment()
zapConfig := zap.NewDevelopmentConfig()
zapConfig.Level.SetLevel(zap.InfoLevel)
if debug {
zapConfig.Level.SetLevel(zap.DebugLevel)
}
logger, err := zapConfig.Build()

if err != nil {
return nil, err
}
Expand Down Expand Up @@ -146,7 +153,7 @@ func (s *Suite) Cleanup(t *testing.T, tt *Test) {

var wg sync.WaitGroup

tt.logger.Info("Stopping all services")
tt.logger.Debug("Stopping all services")
wg.Add(1)
go func() {
defer wg.Done()
Expand All @@ -172,7 +179,7 @@ func (s *Suite) Cleanup(t *testing.T, tt *Test) {
}()

wg.Wait()
tt.logger.Info("All services stopped")
tt.logger.Debug("All services stopped")

tt.resourceManagerSession.ReleaseResources()
}
Loading