fix(installer): Fix more flaky test scenarii (#32984)

test/new-e2e/tests/installer/host/host.go

@@ -167,11 +167,15 @@ func (h *Host) WaitForUnitActive(units ...string) {
 }
 
 // WaitForUnitActivating waits for a systemd unit to be activating
-func (h *Host) WaitForUnitActivating(units ...string) {
+func (h *Host) WaitForUnitActivating(t *testing.T, units ...string) {
 	for _, unit := range units {
 		_, err := h.remote.Execute(fmt.Sprintf("timeout=60; unit=%s; while ! grep -q \"Active: activating\" <(sudo systemctl status $unit) && [ $timeout -gt 0 ]; do sleep 1; ((timeout--)); done; [ $timeout -ne 0 ]", unit))
-		require.NoError(h.t, err, "unit %s did not become activating. logs: %s", unit, h.remote.MustExecute("sudo journalctl -xeu "+unit))
-
+		if err != nil {
+			h.t.Logf("installer logs:\n%s", h.remote.MustExecute("sudo journalctl -xeu datadog-installer"))
+			h.t.Logf("installer exp logs:\n%s", h.remote.MustExecute("sudo journalctl -xeu datadog-installer-exp"))
+			h.t.Logf("unit %s logs:\n%s", unit, h.remote.MustExecute("sudo journalctl -xeu "+unit))
+		}
+		require.NoError(t, err, "unit %s did not become activating")
 	}
 }
 

test/new-e2e/tests/installer/script/all_scripts_test.go

@@ -11,6 +11,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	e2eos "github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/ec2"
@@ -150,8 +151,22 @@ func (s *installerScriptBaseSuite) RunInstallScript(url string, params ...string
 }
 
 func (s *installerScriptBaseSuite) RunInstallScriptWithError(url string, params ...string) error {
+	// Download scripts -- add retries for network issues
+	var err error
+	maxRetries := 5
+	for i := 0; i < maxRetries; i++ {
+		_, err = s.Env().RemoteHost.Execute(fmt.Sprintf("curl -L %s > install_script", url))
+		if err == nil {
+			break
+		}
+		if i == maxRetries-1 {
+			return err
+		}
+		time.Sleep(1 * time.Second)
+	}
+
 	scriptParams := append(params, "DD_API_KEY=test", "DD_INSTALLER_REGISTRY_URL_INSTALLER_PACKAGE=installtesting.datad0g.com")
-	_, err := s.Env().RemoteHost.Execute(fmt.Sprintf("curl -L %s > install_script; %s bash install_script", url, strings.Join(scriptParams, " ")))
+	_, err = s.Env().RemoteHost.Execute(fmt.Sprintf("%s bash install_script", strings.Join(scriptParams, " ")))
 	return err
 }
 

test/new-e2e/tests/installer/unix/package_apm_inject_test.go

@@ -404,7 +404,7 @@ func (s *packageApmInjectSuite) TestInstrumentDockerInactive() {
 	s.RunInstallScript("DD_APM_INSTRUMENTATION_ENABLED=all", "DD_APM_INSTRUMENTATION_LIBRARIES=python", envForceInstall("datadog-agent"))
 	defer s.Purge()
 
-	s.Env().RemoteHost.MustExecute("sudo systemctl start docker")
+	s.host.InstallDocker() // Restart docker cleanly
 
 	s.assertLDPreloadInstrumented(injectOCIPath)
 	s.assertSocketPath()

test/new-e2e/tests/installer/unix/upgrade_scenario_test.go

@@ -482,7 +482,7 @@ func (s *upgradeScenarioSuite) TestConfigUpgradeNewAgents() {
 	timestamp = s.host.LastJournaldTimestamp()
 	s.mustStartConfigExperiment(localCDN.DirPath, datadogAgent, hash)
 	// Assert the successful start of the experiment
-	s.host.WaitForUnitActivating(agentUnitXP)
+	s.host.WaitForUnitActivating(s.T(), agentUnitXP)
 	s.host.WaitForFileExists(false, "/opt/datadog-packages/datadog-agent/experiment/run/agent.pid")
 
 	// Assert experiment is running
@@ -712,7 +712,7 @@ func (s *upgradeScenarioSuite) setCatalog(newCatalog catalog) {
 }
 
 func (s *upgradeScenarioSuite) assertSuccessfulAgentStartExperiment(timestamp host.JournaldTimestamp, version string) {
-	s.host.WaitForUnitActivating(agentUnitXP)
+	s.host.WaitForUnitActivating(s.T(), agentUnitXP)
 	s.host.WaitForFileExists(false, "/opt/datadog-packages/datadog-agent/experiment/run/agent.pid")
 
 	// Assert experiment is running
@@ -826,7 +826,7 @@ func (s *upgradeScenarioSuite) mustStopConfigExperiment(localCDNPath string, pkg
 }
 
 func (s *upgradeScenarioSuite) assertSuccessfulConfigStartExperiment(timestamp host.JournaldTimestamp, hash string) {
-	s.host.WaitForUnitActivating(agentUnitXP)
+	s.host.WaitForUnitActivating(s.T(), agentUnitXP)
 	s.host.WaitForFileExists(false, "/opt/datadog-packages/datadog-agent/experiment/run/agent.pid")
 
 	// Assert experiment is running
@@ -919,7 +919,7 @@ func (s *upgradeScenarioSuite) getInstallerStatus() installerStatus {
 }
 
 func (s *upgradeScenarioSuite) assertSuccessfulInstallerStartExperiment(timestamp host.JournaldTimestamp, version string) {
-	s.host.WaitForUnitActivating(installerUnitXP)
+	s.host.WaitForUnitActivating(s.T(), installerUnitXP)
 	// TODO: check the pid file
 	// s.host.WaitForFileExists(false, "/opt/datadog-packages/datadog-installer/experiment/installer.pid")