setting active proxy very first time to avoid retry at start-up

cmd/tracing-proxy/main.go

@@ -136,11 +136,9 @@ func main() {
 
 	// set proxy details
 	prxy := proxy.NewProxy(c.GetProxyConfig(), opsrampAPI, logsEndpoint, authConfig.Endpoint, c.GetMetricsConfig().OpsRampAPI)
-
-	if err := prxy.UpdateProxyEnvVars(); err != nil {
-		fmt.Printf("error while setting proxy: %v\n", err)
-		os.Exit(1)
-	}
+	prxy.Logger = lgr
+	// connect to working proxy at start-up
+	_ = prxy.SwitchProxy()
 
 	userAgentAddition := "tracing-proxy/" + CollectorVersion
 	upstreamClient, err := libtrace.NewClient(libtrace.ClientConfig{ // nolint:all

logger/logger.go

@@ -27,7 +27,7 @@ type Entry interface {
 }
 
 func GetLoggerImplementation(format, output, filename string, maxSize, maxBackup int, compress bool) Logger {
-	return &LogrusLogger{
+	lgr := &LogrusLogger{
 		LogFormatter: format,
 		LogOutput:    output,
 		File: struct {
@@ -42,4 +42,7 @@ func GetLoggerImplementation(format, output, filename string, maxSize, maxBackup
 			Compress:   compress,
 		},
 	}
+	_ = lgr.Init()
+	_ = lgr.Start()
+	return lgr
 }

logger/logrus.go

@@ -85,6 +85,9 @@ func (l *LogrusLogger) Start() error {
 }
 
 func (l *LogrusLogger) Init() *logrus.Logger {
+	if l.logger != nil {
+		return l.logger
+	}
 	l.logger = logrus.New()
 	return l.logger
 }

metrics/opsramp.go

@@ -157,13 +157,6 @@ func (p *OpsRampMetrics) Start() error {
 				statusCode, err := p.Push()
 				if err != nil {
 					p.Logger.Error().Logf("error while pushing metrics with statusCode: %d and Error: %v", statusCode, err)
-					if strings.Contains(err.Error(), "connection refused") ||
-						strings.Contains(err.Error(), "unreachable") {
-						if p.Proxy.Enabled() {
-							_ = p.Proxy.SwitchProxy()
-						}
-						continue
-					}
 					if err.Error() == missingMetricsWriteScope {
 						p.Logger.Info().Logf("renewing auth token since the existing token is missing metrics:write scope")
 						err := p.RenewOAuthToken()
@@ -549,10 +542,7 @@ func (p *OpsRampMetrics) Populate() {
 
 	_ = p.Proxy.UpdateProxyEnvVars()
 
-	p.Client = http.Client{
-		Transport: utils.CreateNewHTTPTransport(),
-		Timeout:   time.Duration(240) * time.Second,
-	}
+	p.RenewClient()
 }
 
 func ConvertLabelsToMap(labels []prompb.Label) map[string]string {
@@ -854,11 +844,23 @@ func (p *OpsRampMetrics) Push() (int, error) {
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
 		return resp.StatusCode, fmt.Errorf("unexpected status code %d while pushing: %s", resp.StatusCode, body)
 	}
-	p.Logger.Debug().Logf("metrics push response: %v", string(body))
+	p.Logger.Debug().Logf("metrics %s push response: %v", p.prefix, string(body))
 
 	return resp.StatusCode, nil
 }
 
+func (p *OpsRampMetrics) RenewClient() {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.Client.CloseIdleConnections()
+
+	p.Client = http.Client{
+		Transport: utils.CreateNewHTTPTransport(),
+		Timeout:   time.Duration(240) * time.Second,
+	}
+}
+
 func (p *OpsRampMetrics) RenewOAuthToken() error {
 	p.oAuthToken = new(OpsRampAuthTokenResponse)
 
@@ -876,13 +878,24 @@ func (p *OpsRampMetrics) RenewOAuthToken() error {
 
 	resp, err := p.Client.Do(req)
 	if err != nil {
+		retry := false
 		if strings.Contains(err.Error(), "connection refused") ||
 			strings.Contains(err.Error(), "unreachable") {
 			if p.Proxy.Enabled() {
 				_ = p.Proxy.SwitchProxy()
+				p.RenewClient()
+				retry = true
 			}
 		}
-		return err
+
+		if retry {
+			resp, err = p.Client.Do(req)
+			if err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
 	}
 
 	respBody, err := io.ReadAll(resp.Body)
@@ -900,10 +913,24 @@ func (p *OpsRampMetrics) RenewOAuthToken() error {
 }
 
 func (p *OpsRampMetrics) Send(request *http.Request) (*http.Response, error) {
+	retry := false
 	response, err := p.Client.Do(request)
 	if err == nil && response != nil && (response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted) {
 		return response, nil
 	}
+	if err != nil &&
+		(strings.Contains(err.Error(), "connection refused") ||
+			strings.Contains(err.Error(), "unreachable")) {
+		if p.Proxy.Enabled() {
+			_ = p.Proxy.SwitchProxy()
+			p.RenewClient()
+			retry = true
+		}
+	}
+	if retry {
+		response, err = p.Client.Do(request)
+	}
+
 	if response != nil && response.StatusCode == http.StatusProxyAuthRequired { // OpsRamp uses this for bad auth token
 		err := p.RenewOAuthToken()
 		if err != nil {