From c871ec2a1fca7deed89d2680221b3a1b913431be Mon Sep 17 00:00:00 2001 From: Samuel Scard <10246309+SamJamS@users.noreply.github.com> Date: Sat, 20 Apr 2024 00:36:19 +0100 Subject: [PATCH 1/2] sidecar: set to not ready when failing to fetch prometheus version Signed-off-by: Samuel Scard <10246309+SamJamS@users.noreply.github.com> --- cmd/thanos/sidecar.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 74ab3090fa..79535ee25c 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -189,6 +189,8 @@ func runSidecar( "msg", "failed to fetch prometheus version. Is Prometheus running? Retrying", "err", err, ) + promUp.Set(0) + statusProber.NotReady(err) return err } From 12c399c250f27ebc6e94a2e70f07dd3a8a7c926c Mon Sep 17 00:00:00 2001 From: Samuel Scard <10246309+SamJamS@users.noreply.github.com> Date: Sat, 20 Apr 2024 00:37:08 +0100 Subject: [PATCH 2/2] add TestSidecarNotReadyNoPrometheus Signed-off-by: Samuel Scard <10246309+SamJamS@users.noreply.github.com> --- test/e2e/query_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 78b92c3eac..e555bebd73 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -235,6 +235,39 @@ func TestSidecarNotReady(t *testing.T) { })) } +func TestSidecarNotReadyNoPrometheus(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("sidecar-noProm") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + // Start sidecar without ever starting Prometheus. + _, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "alone", e2ethanos.DefaultPromConfig("prom-alone", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "") + testutil.Ok(t, sidecar.Start()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Sidecar should not be ready if there was never a ready Prometheus. + testutil.Ok(t, runutil.Retry(1*time.Second, ctx.Done(), func() (rerr error) { + req, err := http.NewRequestWithContext(ctx, "GET", "http://"+sidecar.Endpoint("http")+"/-/ready", nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer runutil.CloseWithErrCapture(&rerr, resp.Body, "closing resp body") + + if resp.StatusCode == 200 { + return fmt.Errorf("got status code %d", resp.StatusCode) + } + return nil + })) +} + func TestQuery(t *testing.T) { t.Parallel()