From 3442f718275931f824d0b80692b23c4b1edfb6a8 Mon Sep 17 00:00:00 2001 From: Kobi Samoray Date: Wed, 24 Nov 2021 10:34:02 +0200 Subject: [PATCH] IPPool counters Add IPPools usage counters, and expose them via CRD and Prometheus. Signed-off-by: Kobi Samoray --- build/yamls/antrea-aks.yml | 7 + build/yamls/antrea-eks.yml | 7 + build/yamls/antrea-gke.yml | 7 + build/yamls/antrea-ipsec.yml | 7 + build/yamls/antrea-kind.yml | 7 + build/yamls/antrea.yml | 7 + build/yamls/base/crds.yml | 7 + cmd/antrea-controller/controller.go | 6 + pkg/apis/crd/v1alpha2/types.go | 18 +-- .../crd/v1alpha2/zz_generated.deepcopy.go | 33 ++-- pkg/controller/externalippool/controller.go | 2 +- .../externalippool/controller_test.go | 24 +-- pkg/controller/ipam/metrics.go | 149 ++++++++++++++++++ pkg/ipam/ipallocator/allocator.go | 6 +- pkg/ipam/poolallocator/allocator.go | 5 + pkg/ipam/poolallocator/allocator_test.go | 34 +++- 16 files changed, 286 insertions(+), 40 deletions(-) create mode 100644 pkg/controller/ipam/metrics.go diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 10695ecc8f8..3499dbfd885 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 600d1ac7dae..9b2e301a1a7 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index d5a3fc74ea2..81baa158314 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 69122ca2644..abb9951bcfe 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/antrea-kind.yml b/build/yamls/antrea-kind.yml index 1b7ba036a0f..6e6f7afcdfa 100644 --- a/build/yamls/antrea-kind.yml +++ b/build/yamls/antrea-kind.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 773da37195c..e6b8dc7a7b5 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1247,6 +1247,13 @@ spec: type: string type: object type: array + usage: + properties: + total: + type: integer + used: + type: integer + type: object type: object required: - spec diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index eb653d73fd5..7fe3a77c1ef 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -305,6 +305,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 60a726591af..865fbf95068 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -182,6 +182,11 @@ func run(o *Options) error { externalIPController = serviceexternalip.NewServiceExternalIPController(client, serviceInformer, externalIPPoolController) } + var ipamMetricsHandler *antreaipam.AntreaIPAMMetricsHandler + if features.DefaultFeatureGate.Enabled(features.AntreaIPAM) { + ipamMetricsHandler = antreaipam.NewAntreaIPAMMetricsHandler(crdClient, crdInformerFactory) + } + var traceflowController *traceflow.Controller if features.DefaultFeatureGate.Enabled(features.Traceflow) { traceflowController = traceflow.NewTraceflowController(crdClient, podInformer, tfInformer) @@ -310,6 +315,7 @@ func run(o *Options) error { if antreaIPAMController != nil { go antreaIPAMController.Run(stopCh) + go ipamMetricsHandler.Run(stopCh) } <-stopCh diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 0b04bc13ae7..197815acb4c 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -266,14 +266,7 @@ type IPRange struct { } type ExternalIPPoolStatus struct { - Usage ExternalIPPoolUsage `json:"usage,omitempty"` -} - -type ExternalIPPoolUsage struct { - // Total number of IPs. - Total int `json:"total"` - // Number of allocated IPs. - Used int `json:"used"` + Usage IPPoolUsage `json:"usage,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -330,7 +323,14 @@ type SubnetIPRange struct { type IPPoolStatus struct { IPAddresses []IPAddressState `json:"ipAddresses,omitempty"` - // TODO: add usage statistics + Usage IPPoolUsage `json:"usage,omitempty"` +} + +type IPPoolUsage struct { + // Total number of IPs. + Total int `json:"total"` + // Number of allocated IPs. + Used int `json:"used"` } type IPAddressPhase string diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index 4334cb9f36e..335884a422e 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -413,22 +413,6 @@ func (in *ExternalIPPoolStatus) DeepCopy() *ExternalIPPoolStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExternalIPPoolUsage) DeepCopyInto(out *ExternalIPPoolUsage) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalIPPoolUsage. -func (in *ExternalIPPoolUsage) DeepCopy() *ExternalIPPoolUsage { - if in == nil { - return nil - } - out := new(ExternalIPPoolUsage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GroupCondition) DeepCopyInto(out *GroupCondition) { *out = *in @@ -655,6 +639,7 @@ func (in *IPPoolStatus) DeepCopyInto(out *IPPoolStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.Usage = in.Usage return } @@ -668,6 +653,22 @@ func (in *IPPoolStatus) DeepCopy() *IPPoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolUsage) DeepCopyInto(out *IPPoolUsage) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolUsage. +func (in *IPPoolUsage) DeepCopy() *IPPoolUsage { + if in == nil { + return nil + } + out := new(IPPoolUsage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IPRange) DeepCopyInto(out *IPRange) { *out = *in diff --git a/pkg/controller/externalippool/controller.go b/pkg/controller/externalippool/controller.go index 0fd219c6c79..326c9e53ed4 100644 --- a/pkg/controller/externalippool/controller.go +++ b/pkg/controller/externalippool/controller.go @@ -308,7 +308,7 @@ func (c *ExternalIPPoolController) updateExternalIPPoolStatus(poolName string) e var getErr error if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { actualStatus := eip.Status - usage := antreacrds.ExternalIPPoolUsage{Total: total, Used: used} + usage := antreacrds.IPPoolUsage{Total: total, Used: used} if actualStatus.Usage == usage { return nil } diff --git a/pkg/controller/externalippool/controller_test.go b/pkg/controller/externalippool/controller_test.go index 9d7abf8d029..9e4e40e3c0f 100644 --- a/pkg/controller/externalippool/controller_test.go +++ b/pkg/controller/externalippool/controller_test.go @@ -79,7 +79,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom string expectedIP string expectError bool - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "allocate from proper IP pool", @@ -90,7 +90,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip1", expectedIP: "10.10.10.2", expectError: false, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, }, }, @@ -109,7 +109,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip1", expectedIP: "", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 2}, }, }, @@ -122,7 +122,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip2", expectedIP: "", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 0}, }, }, @@ -164,7 +164,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease string ipToRelease string expectError bool - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "release IP to pool", @@ -181,7 +181,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease: "eip1", ipToRelease: "10.10.10.2", expectError: false, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, }, }, @@ -200,7 +200,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease: "eip1", ipToRelease: "10.10.11.2", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 2}, }, }, @@ -455,7 +455,7 @@ func TestIPPoolHasIP(t *testing.T) { } } -func checkExternalIPPoolStatus(t *testing.T, controller *controller, poolName string, expectedStatus antreacrds.ExternalIPPoolUsage) { +func checkExternalIPPoolStatus(t *testing.T, controller *controller, poolName string, expectedStatus antreacrds.IPPoolUsage) { exists := controller.IPPoolExists(poolName) require.True(t, exists) err := wait.PollImmediate(50*time.Millisecond, 2*time.Second, func() (found bool, err error) { @@ -475,7 +475,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { allocations []IPAllocation allocationsToRestore []IPAllocation expectedSucceeded []IPAllocation - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "restore all IP successfully", @@ -516,7 +516,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, {Total: 2, Used: 1}, }, @@ -561,7 +561,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, {Total: 2, Used: 1}, }, @@ -598,7 +598,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 0}, {Total: 2, Used: 1}, }, diff --git a/pkg/controller/ipam/metrics.go b/pkg/controller/ipam/metrics.go new file mode 100644 index 00000000000..d192fed4a92 --- /dev/null +++ b/pkg/controller/ipam/metrics.go @@ -0,0 +1,149 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipam + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/klog/v2" + + crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" + "antrea.io/antrea/pkg/client/clientset/versioned" + crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" + "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" + crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/ipam/poolallocator" +) + +const ( + metricNamespaceAntrea = "antrea" + metricSubsystemIPAM = "ipam" +) + +type AntreaIPAMMetricsHandler struct { + crdClient versioned.Interface + ipPoolLister crdlisters.IPPoolLister + ipPoolInformer v1alpha2.IPPoolInformer +} + +var ( + ipPoolUsageTotal = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Namespace: metricNamespaceAntrea, + Subsystem: metricSubsystemIPAM, + Name: "ippool_usage_total", + Help: "Total number of available IP addresses for the address pool.", + StabilityLevel: metrics.ALPHA, + }, + []string{"pool_name"}, + ) + ipPoolUsageUsed = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Namespace: metricNamespaceAntrea, + Subsystem: metricSubsystemIPAM, + Name: "ippool_usage_used", + Help: "Number of allocated IP addresses for the address pool.", + StabilityLevel: metrics.ALPHA, + }, + []string{"pool_name"}, + ) +) + +func NewAntreaIPAMMetricsHandler(crdClient versioned.Interface, crdInformerFactory crdinformers.SharedInformerFactory) *AntreaIPAMMetricsHandler { + legacyregistry.MustRegister(ipPoolUsageTotal) + legacyregistry.MustRegister(ipPoolUsageUsed) + + ipPoolInformer := crdInformerFactory.Crd().V1alpha2().IPPools() + mh := &AntreaIPAMMetricsHandler{ + crdClient: crdClient, + ipPoolInformer: ipPoolInformer, + ipPoolLister: ipPoolInformer.Lister()} + return mh +} + +func (c *AntreaIPAMMetricsHandler) Run(stopCh <-chan struct{}) { + c.ipPoolInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ + AddFunc: c.createHandler, + UpdateFunc: c.updateHandler, + DeleteFunc: c.deleteHandler, + }, 0) + + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.ipPoolInformer.Informer().HasSynced) { + return + } + <-stopCh +} + +func (c *AntreaIPAMMetricsHandler) createHandler(obj interface{}) { + c.updateIPPoolCounters(obj) +} + +func (c *AntreaIPAMMetricsHandler) updateHandler(oldObj, newObj interface{}) { + c.updateIPPoolCounters(newObj) +} + +func (c *AntreaIPAMMetricsHandler) deleteHandler(obj interface{}) { + ipPool := obj.(*crdv1a2.IPPool) + + if !ipPoolUsageTotal.DeleteLabelValues(ipPool.Name) { + klog.Warningf("Deletion of IPPool %s ippool_usage_total metric failed", ipPool.Name) + } + if !ipPoolUsageUsed.DeleteLabelValues(ipPool.Name) { + klog.Warningf("Deletion of IPPool %s ippool_usage_total metric failed", ipPool.Name) + } +} + +func (c *AntreaIPAMMetricsHandler) updateIPPoolCounters(obj interface{}) { + ipPool := obj.(*crdv1a2.IPPool) + + allocator, err := poolallocator.NewIPPoolAllocator(ipPool.Name, c.crdClient, c.ipPoolLister) + + if err != nil { + klog.Warningf("Failed to initialize allocator for IPPool %s", ipPool.Name) + } + + // Total is fetched from allocator as here are trapped changes to CRD, e.g addition of new IPRange + total := allocator.Total() + + // Used is gathered from IP allocation status within the CRD - as it can be set by each one of the agents + used := len(ipPool.Status.IPAddresses) + + // Set Prometheus metrics + ipPoolUsageTotal.WithLabelValues(ipPool.Name).Set(float64(total)) + ipPoolUsageUsed.WithLabelValues(ipPool.Name).Set(float64(used)) + + // Update the status within the CRD + pool, err := c.ipPoolLister.Get(ipPool.Name) + if err != nil { + klog.Warningf("Failed to retrieve pool %s from CRD", ipPool.Name) + return + } + + // If update has no effect, exit + if pool.Status.Usage.Used == used && pool.Status.Usage.Total == total { + return + } + pool.Status.Usage.Used = used + pool.Status.Usage.Total = total + + _, err = c.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), pool, metav1.UpdateOptions{}) + if err != nil { + klog.Warningf("IP Pool %s update with status %+v failed: %+v", pool.Name, pool.Status, err) + } +} diff --git a/pkg/ipam/ipallocator/allocator.go b/pkg/ipam/ipallocator/allocator.go index 71798394f8d..b8a8ac94bf3 100644 --- a/pkg/ipam/ipallocator/allocator.go +++ b/pkg/ipam/ipallocator/allocator.go @@ -247,6 +247,10 @@ func (a *SingleIPAllocator) Free() int { return a.max - a.count - len(a.reservedIPs) } +func (a *SingleIPAllocator) Total() int { + return a.max - len(a.reservedIPs) +} + // Has returns whether the provided IP is in the range or not. func (a *SingleIPAllocator) Has(ip net.IP) bool { offset := a.getOffset(ip) @@ -322,7 +326,7 @@ func (ma MultiIPAllocator) Free() int { func (ma MultiIPAllocator) Total() int { total := 0 for _, a := range ma { - total += a.max - len(a.reservedIPs) + total += a.Total() } return total } diff --git a/pkg/ipam/poolallocator/allocator.go b/pkg/ipam/poolallocator/allocator.go index 2ea90c39697..ef6e75ff14b 100644 --- a/pkg/ipam/poolallocator/allocator.go +++ b/pkg/ipam/poolallocator/allocator.go @@ -608,3 +608,8 @@ func (a *IPPoolAllocator) getReservedIP(reservedOwner v1alpha2.IPAddressOwner) ( } return nil, nil } + +func (a IPPoolAllocator) Total() int { + _, allocators, _ := a.getPoolAndInitIPAllocators() + return allocators.Total() +} diff --git a/pkg/ipam/poolallocator/allocator_test.go b/pkg/ipam/poolallocator/allocator_test.go index caca0b4dc2b..4d05345cea8 100644 --- a/pkg/ipam/poolallocator/allocator_test.go +++ b/pkg/ipam/poolallocator/allocator_test.go @@ -111,6 +111,7 @@ func TestAllocateIP(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 21, allocator.Total()) // Allocate specific IP from the range returnInfo, err := allocator.AllocateIP(net.ParseIP("10.2.2.101"), crdv1a2.IPAddressPhaseAllocated, fakePodOwner) @@ -133,8 +134,36 @@ func TestAllocateIP(t *testing.T) { func TestAllocateNext(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) + poolName := "fakePool" + ipRange := crdv1a2.IPRange{ + Start: "10.2.2.100", + End: "10.2.2.120", + } + subnetInfo := crdv1a2.SubnetInfo{ + Gateway: "10.2.2.1", + PrefixLength: 24, + } + subnetRange := crdv1a2.SubnetIPRange{IPRange: ipRange, + SubnetInfo: subnetInfo} - poolName := uuid.New().String() + pool := crdv1a2.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: poolName}, + Spec: crdv1a2.IPPoolSpec{IPRanges: []crdv1a2.SubnetIPRange{subnetRange}}, + } + + allocator := newTestIPPoolAllocator(&pool, stopCh) + assert.Equal(t, 21, allocator.Total()) + + validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101"}) +} + +// This test verifies correct behavior in case of update conflict. Allocation should be retiried +// Taking into account the latest status +func TestAllocateConflict(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + poolName := "fakePool" ipRange := crdv1a2.IPRange{ Start: "10.2.2.100", End: "10.2.2.120", @@ -184,6 +213,7 @@ func TestAllocateNextMultiRange(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 16, allocator.Total()) // Allocate the 2 available IPs from first range then switch to second range validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101", "10.2.2.2", "10.2.2.3"}) @@ -219,6 +249,7 @@ func TestAllocateNextMultiRangeExausted(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 3, allocator.Total()) // Allocate all available IPs validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101", "10.2.2.200"}) @@ -295,6 +326,7 @@ func TestReleaseResource(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 15, allocator.Total()) // Allocate the single available IPs from first range then 3 IPs from second range validateAllocationSequence(t, allocator, subnetInfo, []string{"2001::1000", "2001::2", "2001::3", "2001::4", "2001::5"})