From 7e4f4ec5900b28bb16c3ad3867a83a6bfbda683d Mon Sep 17 00:00:00 2001 From: Slavik Panasovets Date: Wed, 15 Feb 2023 16:32:07 +0000 Subject: [PATCH] Add new metric status PersistentError for L4 DualStack Report PersistentError if service is in Error after 20 minutes of retries --- pkg/l4lb/l4controller_test.go | 26 ++++ pkg/loadbalancers/l4.go | 6 +- pkg/loadbalancers/l4netlb.go | 2 +- pkg/metrics/l4metrics_test.go | 227 +++++++++++++++++++++++++++------- pkg/metrics/metrics.go | 41 ++++-- pkg/metrics/types.go | 17 ++- 6 files changed, 256 insertions(+), 63 deletions(-) diff --git a/pkg/l4lb/l4controller_test.go b/pkg/l4lb/l4controller_test.go index a75ea16c5f..5c2647b8e8 100644 --- a/pkg/l4lb/l4controller_test.go +++ b/pkg/l4lb/l4controller_test.go @@ -638,6 +638,32 @@ func TestProcessDualStackServiceOnUserError(t *testing.T) { } } +func TestDualStackILBStatusForErrorSync(t *testing.T) { + l4c := newServiceController(t, newFakeGCEWithUserNoIPv6SubnetError()) + l4c.enableDualStack = true + (l4c.ctx.Cloud.Compute().(*cloud.MockGCE)).MockForwardingRules.InsertHook = mock.InsertForwardingRulesInternalErrHook + + newSvc := test.NewL4ILBDualStackService(8080, api_v1.ProtocolTCP, []api_v1.IPFamily{api_v1.IPv4Protocol, api_v1.IPv6Protocol}, api_v1.ServiceExternalTrafficPolicyTypeCluster) + addILBService(l4c, newSvc) + addNEG(l4c, newSvc) + syncResult := l4c.processServiceCreateOrUpdate(newSvc) + if syncResult.Error == nil { + t.Fatalf("Failed to generate error when syncing service %s", newSvc.Name) + } + if syncResult.MetricsState.IsUserError { + t.Errorf("syncResult.MetricsState.IsUserError should be false, got true") + } + if syncResult.MetricsState.InSuccess { + t.Errorf("syncResult.MetricsState.InSuccess should be false, got true") + } + if syncResult.DualStackMetricsState.Status != metrics.StatusError { + t.Errorf("syncResult.DualStackMetricsState.Status should be %s, got %s", metrics.StatusError, syncResult.DualStackMetricsState.Status) + } + if syncResult.DualStackMetricsState.FirstSyncErrorTime == nil { + t.Fatalf("Metric status FirstSyncErrorTime for service %s/%s mismatch, expected: not nil, received: nil", newSvc.Namespace, newSvc.Name) + } +} + func TestProcessUpdateILBIPFamilies(t *testing.T) { testCases := []struct { desc string diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index 579cab3cc0..1c51cd115f 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -283,11 +283,12 @@ func (l4 *L4) getFRNameWithProtocol(protocol string) string { func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) *L4ILBSyncResult { l4.Service = svc + startTime := time.Now() result := &L4ILBSyncResult{ Annotations: make(map[string]string), - StartTime: time.Now(), + StartTime: startTime, SyncType: SyncTypeCreate, - DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc), + DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc, &startTime), } // If service already has an IP assigned, treat it as an update instead of a new Loadbalancer. @@ -388,6 +389,7 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service } if l4.enableDualStack { result.DualStackMetricsState.Status = metrics.StatusSuccess + result.DualStackMetricsState.FirstSyncErrorTime = nil } return result } diff --git a/pkg/loadbalancers/l4netlb.go b/pkg/loadbalancers/l4netlb.go index 92c9c0e13a..1d2dcbb8b9 100644 --- a/pkg/loadbalancers/l4netlb.go +++ b/pkg/loadbalancers/l4netlb.go @@ -74,7 +74,7 @@ func NewL4SyncResult(syncType string, svc *corev1.Service) *L4NetLBSyncResult { StartTime: startTime, SyncType: syncType, MetricsState: metrics.InitL4NetLBServiceState(&startTime), - DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc), + DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc, &startTime), } return result } diff --git a/pkg/metrics/l4metrics_test.go b/pkg/metrics/l4metrics_test.go index c8cee637af..b52eb037fa 100644 --- a/pkg/metrics/l4metrics_test.go +++ b/pkg/metrics/l4metrics_test.go @@ -381,23 +381,28 @@ func TestComputeL4NetLBMetrics(t *testing.T) { func TestComputeL4NetLBDualStackMetrics(t *testing.T) { t.Parallel() + + currTime := time.Now() + before10min := currTime.Add(-10 * time.Minute) + before20min := currTime.Add(-20 * time.Minute) + for _, tc := range []struct { desc string serviceStates []L4DualStackServiceState - expectL4NetLBDualStackCount map[L4DualStackServiceState]int + expectL4NetLBDualStackCount map[L4DualStackServiceCount]int }{ { desc: "empty input", serviceStates: []L4DualStackServiceState{}, - expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{}, + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{}, }, { desc: "one l4 NetLB dual-stack service", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess), + newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess, nil), }, - expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusSuccess, @@ -407,23 +412,49 @@ func TestComputeL4NetLBDualStackMetrics(t *testing.T) { { desc: "l4 NetLB dual-stack service in error state", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4", "SingleStack", StatusError), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, nil), }, - expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusError, }: 1, }, }, + { + desc: "l4 NetLB dual-stack service in error state, for 10 minutes", + serviceStates: []L4DualStackServiceState{ + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before10min), + }, + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + }, + }, + { + desc: "l4 NetLB dual-stack service in error state, for 20 minutes", + serviceStates: []L4DualStackServiceState{ + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before20min), + }, + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusPersistentError, + }: 1, + }, + }, { desc: "L4 NetLB dual-stack service with IPv4,IPv6 Families", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), }, - expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4,IPv6", "RequireDualStack", StatusSuccess, @@ -433,28 +464,35 @@ func TestComputeL4NetLBDualStackMetrics(t *testing.T) { { desc: "many l4 NetLB dual-stack services", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), - newL4DualStackServiceState("IPv4", "SingleStack", StatusError), - newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess), - newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess), - }, - expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, nil), + newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before10min), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before20min), + }, + expectL4NetLBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4,IPv6", "RequireDualStack", StatusSuccess, }: 2, - L4DualStackServiceState{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusError, - }: 1, - L4DualStackServiceState{ + }: 2, + L4DualStackServiceCount{ "IPv6", "SingleStack", StatusSuccess, }: 2, + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusPersistentError, + }: 1, }, }, } { @@ -514,6 +552,58 @@ func TestRetryPeriodForL4NetLBServices(t *testing.T) { } } +func TestRetryPeriodForL4ILBDualStackServices(t *testing.T) { + t.Parallel() + currTime := time.Now() + before5min := currTime.Add(-5 * time.Minute) + + svcName1 := "svc1" + metrics := FakeControllerMetrics() + + errorState := newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &currTime) + metrics.SetL4ILBDualStackService(svcName1, errorState) + + // change FirstSyncErrorTime and verify it will not change metrics state + errorState.FirstSyncErrorTime = &before5min + metrics.SetL4ILBDualStackService(svcName1, errorState) + state, ok := metrics.l4ILBDualStackServiceMap[svcName1] + if !ok { + t.Fatalf("state should be set") + } + if *state.FirstSyncErrorTime != currTime { + t.Errorf("FirstSyncErrorTime should not change, expected %v, got %v", currTime, *state.FirstSyncErrorTime) + } + if state.Status != StatusError { + t.Errorf("Expected status %s, got %s", StatusError, state.Status) + } +} + +func TestRetryPeriodForL4NetLBDualStackServices(t *testing.T) { + t.Parallel() + currTime := time.Now() + before5min := currTime.Add(-5 * time.Minute) + + svcName1 := "svc1" + metrics := FakeControllerMetrics() + + errorState := newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &currTime) + metrics.SetL4NetLBDualStackService(svcName1, errorState) + + // change FirstSyncErrorTime and verify it will not change metrics state + errorState.FirstSyncErrorTime = &before5min + metrics.SetL4NetLBDualStackService(svcName1, errorState) + state, ok := metrics.l4NetLBDualStackServiceMap[svcName1] + if !ok { + t.Fatalf("state should be set") + } + if *state.FirstSyncErrorTime != currTime { + t.Errorf("FirstSyncErrorTime should not change, expected %v, got %v", currTime, *state.FirstSyncErrorTime) + } + if state.Status != StatusError { + t.Errorf("Expected status %s, got %s", StatusError, state.Status) + } +} + func checkMetricsComputation(newMetrics *ControllerMetrics, expErrorCount, expSvcCount int) error { got := newMetrics.computeL4NetLBMetrics() if got.inError != expErrorCount { @@ -527,23 +617,28 @@ func checkMetricsComputation(newMetrics *ControllerMetrics, expErrorCount, expSv func TestComputeL4ILBDualStackMetrics(t *testing.T) { t.Parallel() + + currTime := time.Now() + before10min := currTime.Add(-10 * time.Minute) + before20min := currTime.Add(-20 * time.Minute) + for _, tc := range []struct { desc string serviceStates []L4DualStackServiceState - expectL4ILBDualStackCount map[L4DualStackServiceState]int + expectL4ILBDualStackCount map[L4DualStackServiceCount]int }{ { desc: "empty input", serviceStates: []L4DualStackServiceState{}, - expectL4ILBDualStackCount: map[L4DualStackServiceState]int{}, + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{}, }, { desc: "one l4 ilb dual-stack service", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess), + newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess, nil), }, - expectL4ILBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusSuccess, @@ -553,23 +648,49 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) { { desc: "l4 ilb dual-stack service in error state", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4", "SingleStack", StatusError), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, nil), + }, + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + }, + }, + { + desc: "l4 ilb dual-stack service in error state, for 10 minutes", + serviceStates: []L4DualStackServiceState{ + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before10min), }, - expectL4ILBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusError, }: 1, }, }, + { + desc: "l4 ilb dual-stack service in error state, for 20 minutes", + serviceStates: []L4DualStackServiceState{ + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before20min), + }, + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusPersistentError, + }: 1, + }, + }, { desc: "L4 ILB dual-stack service with IPv4,IPv6 Families", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), }, - expectL4ILBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4,IPv6", "RequireDualStack", StatusSuccess, @@ -579,28 +700,35 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) { { desc: "many l4 ilb dual-stack services", serviceStates: []L4DualStackServiceState{ - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), - newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), - newL4DualStackServiceState("IPv4", "SingleStack", StatusError), - newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess), - newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess), - }, - expectL4ILBDualStackCount: map[L4DualStackServiceState]int{ - L4DualStackServiceState{ + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, nil), + newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess, nil), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before10min), + newL4DualStackServiceState("IPv4", "SingleStack", StatusError, &before20min), + }, + expectL4ILBDualStackCount: map[L4DualStackServiceCount]int{ + L4DualStackServiceCount{ "IPv4,IPv6", "RequireDualStack", StatusSuccess, }: 2, - L4DualStackServiceState{ + L4DualStackServiceCount{ "IPv4", "SingleStack", StatusError, - }: 1, - L4DualStackServiceState{ + }: 2, + L4DualStackServiceCount{ "IPv6", "SingleStack", StatusSuccess, }: 2, + L4DualStackServiceCount{ + "IPv4", + "SingleStack", + StatusPersistentError, + }: 1, }, }, } { @@ -619,10 +747,13 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) { } } -func newL4DualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStatus) L4DualStackServiceState { +func newL4DualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStatus, firstSyncErrorTime *time.Time) L4DualStackServiceState { return L4DualStackServiceState{ - IPFamilies: ipFamilies, - IPFamilyPolicy: ipFamilyPolicy, - Status: status, + L4DualStackServiceCount: L4DualStackServiceCount{ + IPFamilies: ipFamilies, + IPFamilyPolicy: ipFamilyPolicy, + Status: status, + }, + FirstSyncErrorTime: firstSyncErrorTime, } } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a6963ec4a1..e90c98b173 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -39,7 +39,8 @@ const ( // Env variable for ingress version versionVar = "INGRESS_VERSION" // Dummy float so we can used bool based timeseries - versionValue = 1.0 + versionValue = 1.0 + persistentErrorThreshold = 20 * time.Minute ) var ( @@ -308,6 +309,12 @@ func (im *ControllerMetrics) SetL4ILBDualStackService(svcKey string, state L4Dua if im.l4ILBDualStackServiceMap == nil { klog.Fatalf("L4 ILB DualStack Metrics failed to initialize correctly.") } + if state.Status == StatusError { + if previousState, ok := im.l4ILBDualStackServiceMap[svcKey]; ok && previousState.FirstSyncErrorTime != nil { + // If service is in Error state and retry timestamp was set then do not update it. + state.FirstSyncErrorTime = previousState.FirstSyncErrorTime + } + } im.l4ILBDualStackServiceMap[svcKey] = state } @@ -353,6 +360,13 @@ func (im *ControllerMetrics) SetL4NetLBDualStackService(svcKey string, state L4D if im.l4NetLBDualStackServiceMap == nil { klog.Fatalf("L4 NetLB DualStack Metrics failed to initialize correctly.") } + + if state.Status == StatusError { + if previousState, ok := im.l4NetLBDualStackServiceMap[svcKey]; ok && previousState.FirstSyncErrorTime != nil { + // If service is in Error state and retry timestamp was set then do not update it. + state.FirstSyncErrorTime = previousState.FirstSyncErrorTime + } + } im.l4NetLBDualStackServiceMap[svcKey] = state } @@ -613,15 +627,20 @@ func (im *ControllerMetrics) computeL4ILBMetrics() map[feature]int { } // computeL4ILBDualStackMetrics aggregates L4 ILB DualStack metrics in the cache. -func (im *ControllerMetrics) computeL4ILBDualStackMetrics() map[L4DualStackServiceState]int { +func (im *ControllerMetrics) computeL4ILBDualStackMetrics() map[L4DualStackServiceCount]int { im.Lock() defer im.Unlock() klog.V(4).Infof("Computing L4 DualStack ILB usage metrics from service state map: %#v", im.l4ILBDualStackServiceMap) - counts := map[L4DualStackServiceState]int{} + counts := map[L4DualStackServiceCount]int{} for key, state := range im.l4ILBDualStackServiceMap { klog.V(6).Infof("ILB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status) - counts[state]++ + if state.Status == StatusError && + state.FirstSyncErrorTime != nil && + time.Since(*state.FirstSyncErrorTime) > persistentErrorThreshold { + state.Status = StatusPersistentError + } + counts[state.L4DualStackServiceCount]++ } klog.V(4).Info("L4 ILB usage metrics computed.") return counts @@ -662,15 +681,20 @@ func (im *ControllerMetrics) computeL4NetLBMetrics() netLBFeatureCount { } // computeL4NetLBDualStackMetrics aggregates L4 NetLB DualStack metrics in the cache. -func (im *ControllerMetrics) computeL4NetLBDualStackMetrics() map[L4DualStackServiceState]int { +func (im *ControllerMetrics) computeL4NetLBDualStackMetrics() map[L4DualStackServiceCount]int { im.Lock() defer im.Unlock() klog.V(4).Infof("Computing L4 DualStack NetLB usage metrics from service state map: %#v", im.l4NetLBDualStackServiceMap) - counts := map[L4DualStackServiceState]int{} + counts := map[L4DualStackServiceCount]int{} for key, state := range im.l4NetLBDualStackServiceMap { klog.V(6).Infof("NetLB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status) - counts[state]++ + if state.Status == StatusError && + state.FirstSyncErrorTime != nil && + time.Since(*state.FirstSyncErrorTime) > persistentErrorThreshold { + state.Status = StatusPersistentError + } + counts[state.L4DualStackServiceCount]++ } klog.V(4).Info("L4 NetLB usage metrics computed.") return counts @@ -785,7 +809,7 @@ func recordComponentVersion() { componentVersion.WithLabelValues(v).Set(versionValue) } -func InitServiceDualStackMetricsState(svc *corev1.Service) L4DualStackServiceState { +func InitServiceDualStackMetricsState(svc *corev1.Service, startTime *time.Time) L4DualStackServiceState { state := L4DualStackServiceState{} var ipFamiliesStrings []string @@ -801,5 +825,6 @@ func InitServiceDualStackMetricsState(svc *corev1.Service) L4DualStackServiceSta // Always init status with error, and update with Success when service was provisioned state.Status = StatusError + state.FirstSyncErrorTime = startTime return state } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index 14bd1b9369..1ce9c77ffe 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -78,20 +78,29 @@ type L4DualStackServiceStatus string const StatusSuccess = L4DualStackServiceStatus("Success") const StatusUserError = L4DualStackServiceStatus("UserError") const StatusError = L4DualStackServiceStatus("Error") +const StatusPersistentError = L4DualStackServiceStatus("PersistentError") -// L4DualStackServiceState defines ipFamilies, ipFamilyPolicy and status +// L4DualStackServiceCount defines ipFamilies, ipFamilyPolicy and status // of L4 ILB DualStack service -type L4DualStackServiceState struct { +type L4DualStackServiceCount struct { // IPFamilies stores spec.ipFamilies of Service IPFamilies string // IPFamilyPolicy specifies spec.IPFamilyPolicy of Service IPFamilyPolicy string - // Status specifies status of L4 DualStack Service + // Status specifies status of L4 ILB DualStack Status L4DualStackServiceStatus } +// L4DualStackServiceState defines ipFamilies, ipFamilyPolicy, status and tracks +// FirstSyncErrorTime of L4 ILB DualStack service +type L4DualStackServiceState struct { + L4DualStackServiceCount + // FirstSyncErrorTime specifies the time timestamp when the service sync ended up with error for the first time. + FirstSyncErrorTime *time.Time +} + // L4NetLBServiceState defines if network tier is premium and -// if static ip address is managed bu controller +// if static ip address is managed by controller // for an L4 NetLB service. type L4NetLBServiceState struct { // IsManagedIP specifies if Static IP is managed by controller.