[8.16](backport #5950) [proxytest] Wait all requests to finish before…

… closing the server (#5970)

* (backport #5878) proxytest: fix log after test finished (#5878)

(cherry picked from commit 02fb75e)

---------

Co-authored-by: Anderson Queiroz <anderson.queiroz@elastic.co>

* (backport #5950) [proxytest] Await requests before server shutdown #5970

(cherry picked from commit e423d73)

---------

Co-authored-by: Anderson Queiroz <anderson.queiroz@elastic.co>

testing/proxytest/proxytest.go

@@ -38,6 +38,7 @@ type Proxy struct {
 	// proxiedRequests is a "request log" for every request the proxy receives.
 	proxiedRequests   []string
 	proxiedRequestsMu sync.Mutex
+	requestsWG        *sync.WaitGroup
 
 	opts options
 	log  *slog.Logger
@@ -164,8 +165,9 @@ func New(t *testing.T, optns ...Option) *Proxy {
 		lv = slog.LevelDebug
 	}
 	p := Proxy{
-		opts:   opts,
-		client: opts.client,
+		requestsWG: &sync.WaitGroup{},
+		opts:       opts,
+		client:     opts.client,
 		log: slog.New(slog.NewTextHandler(logfWriter(opts.logFn), &slog.HandlerOptions{
 			Level: lv,
 		})),
@@ -176,6 +178,12 @@ func New(t *testing.T, optns ...Option) *Proxy {
 
 	p.Server = httptest.NewUnstartedServer(
 		http.HandlerFunc(func(ww http.ResponseWriter, r *http.Request) {
+			// Sometimes, on CI obviously, the last log happens after the test
+			// finishes. See https://github.com/elastic/elastic-agent/issues/5869.
+			// Therefore, let's add an extra layer to try to avoid that.
+			p.requestsWG.Add(1)
+			defer p.requestsWG.Done()
+
 			w := &proxyResponseWriter{w: ww}
 
 			requestID := uuid.Must(uuid.NewV4()).String()
@@ -187,7 +195,7 @@ func New(t *testing.T, optns ...Option) *Proxy {
 
 			p.ServeHTTP(w, rrr)
 
-			opts.logFn(fmt.Sprintf("[%s] DONE %d - %s %s %s %s\n",
+			p.log.Info(fmt.Sprintf("[%s] DONE %d - %s %s %s %s\n",
 				requestID, w.statusCode, r.Method, r.URL, r.Proto, r.RemoteAddr))
 		}),
 	)
@@ -245,6 +253,15 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	p.serveHTTP(w, r)
 }
 
+func (p *Proxy) Close() {
+	// Sometimes, on CI obviously, the last log happens after the test
+	// finishes. See https://github.com/elastic/elastic-agent/issues/5869.
+	// So, manually wait all ongoing requests to finish.
+	p.requestsWG.Wait()
+
+	p.Server.Close()
+}
+
 func (p *Proxy) serveHTTP(w http.ResponseWriter, r *http.Request) {
 	resp, err := p.processRequest(r)
 	if err != nil {

testing/proxytest/proxytest_test.go

@@ -233,8 +233,10 @@ func TestProxy(t *testing.T) {
 func TestHTTPSProxy(t *testing.T) {
 	targetHost := "not-a-server.co"
 	proxy, client, target := prepareMTLSProxyAndTargetServer(t, targetHost)
-	defer proxy.Close()
-	defer target.Close()
+	t.Cleanup(func() {
+		proxy.Close()
+		target.Close()
+	})
 
 	tcs := []struct {
 		name   string