[Elastic Agent] Add verification check when updating communication to…

… Kibana. (#24489) (#24520)

* Add verification check when updating communication to Kibana.

* Add changelog.

* Add const.

(cherry picked from commit ad3300d)

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -43,6 +43,7 @@
 - Fix bad substitution of API key. {pull}24036[24036]
 - Fix docker enrollment issue related to Fleet Server change. {pull}24155[24155]
 - Improve log on failure of Endpoint Security installation. {pull}24429[24429]
+- Verify communication to Kibana before updating Fleet client. {pull}24489[24489]
 
 ==== New features
 

x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go

@@ -180,14 +180,20 @@ func (f *fleetGateway) worker() {
 				actions[idx] = a
 			}
 
+			var errMsg string
 			if err := f.dispatcher.Dispatch(f.acker, actions...); err != nil {
-				msg := fmt.Sprintf("failed to dispatch actions, error: %s", err)
-				f.log.Error(msg)
-				f.statusReporter.Update(state.Degraded, msg)
+				errMsg = fmt.Sprintf("failed to dispatch actions, error: %s", err)
+				f.log.Error(errMsg)
+				f.statusReporter.Update(state.Failed, errMsg)
 			}
 
 			f.log.Debugf("FleetGateway is sleeping, next update in %s", f.settings.Duration)
-			f.statusReporter.Update(state.Healthy, "")
+			if errMsg != "" {
+				f.statusReporter.Update(state.Failed, errMsg)
+			} else {
+				f.statusReporter.Update(state.Healthy, "")
+			}
+
 		case <-f.bgContext.Done():
 			f.stop()
 			return

x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"sort"
+	"time"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 
@@ -24,6 +25,10 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 )
 
+const (
+	apiStatusTimeout = 15 * time.Second
+)
+
 type clientSetter interface {
 	SetClient(clienter)
 }
@@ -50,7 +55,7 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA
 	}
 
 	h.log.Debugf("handlerPolicyChange: emit configuration for action %+v", a)
-	err = h.handleKibanaHosts(c)
+	err = h.handleKibanaHosts(ctx, c)
 	if err != nil {
 		return err
 	}
@@ -61,7 +66,12 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA
 	return acker.Ack(ctx, action)
 }
 
-func (h *handlerPolicyChange) handleKibanaHosts(c *config.Config) (err error) {
+func (h *handlerPolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config) (err error) {
+	// do not update kibana host from policy; no setters provided with local Fleet Server
+	if len(h.setters) == 0 {
+		return nil
+	}
+
 	cfg, err := configuration.NewFromConfig(c)
 	if err != nil {
 		return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
@@ -94,6 +104,14 @@ func (h *handlerPolicyChange) handleKibanaHosts(c *config.Config) (err error) {
 			err, "fail to create API client with updated hosts",
 			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
 	}
+	ctx, cancel := context.WithTimeout(ctx, apiStatusTimeout)
+	defer cancel()
+	_, err = client.Send(ctx, "GET", "/api/status", nil, nil, nil)
+	if err != nil {
+		return errors.New(
+			err, "fail to communicate with updated API client hosts",
+			errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts))
+	}
 	reader, err := fleetToReader(h.agentInfo, h.config)
 	if err != nil {
 		return errors.New(