diff --git a/build/charts/antrea/crds/ippool.yaml b/build/charts/antrea/crds/ippool.yaml index 41ba5d53092..ae0d4b8f995 100644 --- a/build/charts/antrea/crds/ippool.yaml +++ b/build/charts/antrea/crds/ippool.yaml @@ -100,6 +100,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index ffdbe7d4304..35cf2652a8c 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1370,6 +1370,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 64f7a1a2b62..7629b750118 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -1355,6 +1355,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 7c64611d3a5..5130ec08550 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1370,6 +1370,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 3a5dbc75ac4..6902a44ac29 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1370,6 +1370,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 06a718aec3c..2afc78347bc 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1370,6 +1370,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 69e328f1dd2..9732e21ae55 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1370,6 +1370,13 @@ spec: type: string type: object type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object type: object subresources: status: {} diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 15d18e2b926..39232ea1a81 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -266,14 +266,7 @@ type IPRange struct { } type ExternalIPPoolStatus struct { - Usage ExternalIPPoolUsage `json:"usage,omitempty"` -} - -type ExternalIPPoolUsage struct { - // Total number of IPs. - Total int `json:"total"` - // Number of allocated IPs. - Used int `json:"used"` + Usage IPPoolUsage `json:"usage,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -337,9 +330,15 @@ type SubnetIPRange struct { type IPPoolStatus struct { IPAddresses []IPAddressState `json:"ipAddresses,omitempty"` - // TODO: add usage statistics + Usage IPPoolUsage `json:"usage,omitempty"` } +type IPPoolUsage struct { + // Total number of IPs. + Total int `json:"total"` + // Number of allocated IPs. + Used int `json:"used"` +} type IPAddressPhase string const ( diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index ab1449891e8..9ee9725dd92 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -449,22 +449,6 @@ func (in *ExternalIPPoolStatus) DeepCopy() *ExternalIPPoolStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExternalIPPoolUsage) DeepCopyInto(out *ExternalIPPoolUsage) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalIPPoolUsage. -func (in *ExternalIPPoolUsage) DeepCopy() *ExternalIPPoolUsage { - if in == nil { - return nil - } - out := new(ExternalIPPoolUsage) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GRETunnel) DeepCopyInto(out *GRETunnel) { *out = *in @@ -712,6 +696,7 @@ func (in *IPPoolStatus) DeepCopyInto(out *IPPoolStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.Usage = in.Usage return } @@ -725,6 +710,22 @@ func (in *IPPoolStatus) DeepCopy() *IPPoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolUsage) DeepCopyInto(out *IPPoolUsage) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolUsage. +func (in *IPPoolUsage) DeepCopy() *IPPoolUsage { + if in == nil { + return nil + } + out := new(IPPoolUsage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IPRange) DeepCopyInto(out *IPRange) { *out = *in diff --git a/pkg/controller/externalippool/controller.go b/pkg/controller/externalippool/controller.go index 0fd219c6c79..326c9e53ed4 100644 --- a/pkg/controller/externalippool/controller.go +++ b/pkg/controller/externalippool/controller.go @@ -308,7 +308,7 @@ func (c *ExternalIPPoolController) updateExternalIPPoolStatus(poolName string) e var getErr error if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { actualStatus := eip.Status - usage := antreacrds.ExternalIPPoolUsage{Total: total, Used: used} + usage := antreacrds.IPPoolUsage{Total: total, Used: used} if actualStatus.Usage == usage { return nil } diff --git a/pkg/controller/externalippool/controller_test.go b/pkg/controller/externalippool/controller_test.go index 9d7abf8d029..9e4e40e3c0f 100644 --- a/pkg/controller/externalippool/controller_test.go +++ b/pkg/controller/externalippool/controller_test.go @@ -79,7 +79,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom string expectedIP string expectError bool - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "allocate from proper IP pool", @@ -90,7 +90,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip1", expectedIP: "10.10.10.2", expectError: false, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, }, }, @@ -109,7 +109,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip1", expectedIP: "", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 2}, }, }, @@ -122,7 +122,7 @@ func TestAllocateIPFromPool(t *testing.T) { allocateFrom: "eip2", expectedIP: "", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 0}, }, }, @@ -164,7 +164,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease string ipToRelease string expectError bool - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "release IP to pool", @@ -181,7 +181,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease: "eip1", ipToRelease: "10.10.10.2", expectError: false, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, }, }, @@ -200,7 +200,7 @@ func TestReleaseIP(t *testing.T) { ipPoolToRelease: "eip1", ipToRelease: "10.10.11.2", expectError: true, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 2}, }, }, @@ -455,7 +455,7 @@ func TestIPPoolHasIP(t *testing.T) { } } -func checkExternalIPPoolStatus(t *testing.T, controller *controller, poolName string, expectedStatus antreacrds.ExternalIPPoolUsage) { +func checkExternalIPPoolStatus(t *testing.T, controller *controller, poolName string, expectedStatus antreacrds.IPPoolUsage) { exists := controller.IPPoolExists(poolName) require.True(t, exists) err := wait.PollImmediate(50*time.Millisecond, 2*time.Second, func() (found bool, err error) { @@ -475,7 +475,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { allocations []IPAllocation allocationsToRestore []IPAllocation expectedSucceeded []IPAllocation - expectedIPPoolStatus []antreacrds.ExternalIPPoolUsage + expectedIPPoolStatus []antreacrds.IPPoolUsage }{ { name: "restore all IP successfully", @@ -516,7 +516,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, {Total: 2, Used: 1}, }, @@ -561,7 +561,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 1}, {Total: 2, Used: 1}, }, @@ -598,7 +598,7 @@ func TestExternalIPPoolController_RestoreIPAllocations(t *testing.T) { net.ParseIP("10.10.11.2"), }, }, - expectedIPPoolStatus: []antreacrds.ExternalIPPoolUsage{ + expectedIPPoolStatus: []antreacrds.IPPoolUsage{ {Total: 2, Used: 0}, {Total: 2, Used: 1}, }, diff --git a/pkg/controller/ipam/antrea_ipam_controller.go b/pkg/controller/ipam/antrea_ipam_controller.go index 14b4b597905..7cd22a01b87 100644 --- a/pkg/controller/ipam/antrea_ipam_controller.go +++ b/pkg/controller/ipam/antrea_ipam_controller.go @@ -362,6 +362,49 @@ func (c *AntreaIPAMController) processNextStatefulSetWorkItem() bool { return true } +func (c *AntreaIPAMController) updateIPPoolCounters(obj interface{}) { + ipPool := obj.(*crdv1a2.IPPool) + + allocator, err := poolallocator.NewIPPoolAllocator(ipPool.Name, c.crdClient, c.ipPoolLister) + + if err != nil { + klog.Warningf("Failed to initialize allocator for IPPool %s", ipPool.Name) + } + + // Total is fetched from allocator as here are trapped changes to CRD, e.g addition of new IPRange + total := allocator.Total() + + // Used is gathered from IP allocation status within the CRD - as it can be set by each one of the agents + used := len(ipPool.Status.IPAddresses) + + // Update the status within the CRD + pool, err := c.ipPoolLister.Get(ipPool.Name) + if err != nil { + klog.Warningf("Failed to retrieve pool %s from CRD", ipPool.Name) + return + } + + // If update has no effect, exit + if pool.Status.Usage.Used == used && pool.Status.Usage.Total == total { + return + } + pool.Status.Usage.Used = used + pool.Status.Usage.Total = total + + _, err = c.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), pool, metav1.UpdateOptions{}) + if err != nil { + klog.Warningf("IP Pool %s update with status %+v failed: %+v", pool.Name, pool.Status, err) + } +} + +func (c *AntreaIPAMController) createHandler(obj interface{}) { + c.updateIPPoolCounters(obj) +} + +func (c *AntreaIPAMController) updateHandler(oldObj, newObj interface{}) { + c.updateIPPoolCounters(newObj) +} + // Run begins watching and syncing of a AntreaIPAMController. func (c *AntreaIPAMController) Run(stopCh <-chan struct{}) { @@ -370,6 +413,10 @@ func (c *AntreaIPAMController) Run(stopCh <-chan struct{}) { klog.InfoS("Starting", "controller", controllerName) defer klog.InfoS("Shutting down", "controller", controllerName) + c.ipPoolInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.createHandler, + }) + cacheSyncs := []cache.InformerSynced{c.namespaceListerSynced, c.podInformerSynced, c.statefulSetListerSynced, c.ipPoolListerSynced} if !cache.WaitForNamedCacheSync(controllerName, stopCh, cacheSyncs...) { return diff --git a/pkg/ipam/ipallocator/allocator.go b/pkg/ipam/ipallocator/allocator.go index 71798394f8d..b02fb4928dd 100644 --- a/pkg/ipam/ipallocator/allocator.go +++ b/pkg/ipam/ipallocator/allocator.go @@ -247,6 +247,11 @@ func (a *SingleIPAllocator) Free() int { return a.max - a.count - len(a.reservedIPs) } +// Total returns the number total of IPs within the pool. +func (a *SingleIPAllocator) Total() int { + return a.max - len(a.reservedIPs) +} + // Has returns whether the provided IP is in the range or not. func (a *SingleIPAllocator) Has(ip net.IP) bool { offset := a.getOffset(ip) @@ -322,7 +327,7 @@ func (ma MultiIPAllocator) Free() int { func (ma MultiIPAllocator) Total() int { total := 0 for _, a := range ma { - total += a.max - len(a.reservedIPs) + total += a.Total() } return total } diff --git a/pkg/ipam/poolallocator/allocator.go b/pkg/ipam/poolallocator/allocator.go index fe7b5a9a1b4..e0aa7328e70 100644 --- a/pkg/ipam/poolallocator/allocator.go +++ b/pkg/ipam/poolallocator/allocator.go @@ -145,6 +145,7 @@ func (a *IPPoolAllocator) appendPoolUsage(ipPool *v1alpha2.IPPool, ip net.IP, st } newPool.Status.IPAddresses = append(newPool.Status.IPAddresses, usageEntry) + a.updateUsage(newPool, 1) _, err := a.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), newPool, metav1.UpdateOptions{}) if err != nil { klog.Warningf("IP Pool %s update with status %+v failed: %+v", newPool.Name, newPool.Status, err) @@ -172,6 +173,7 @@ func (a *IPPoolAllocator) updateIPAddressState(ipPool *v1alpha2.IPPool, ip net.I if !found { return fmt.Errorf("ip %s usage not found in pool %s", ipString, newPool.Name) } + a.updateUsage(newPool, 2) _, err := a.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), newPool, metav1.UpdateOptions{}) if err != nil { @@ -202,6 +204,7 @@ func (a *IPPoolAllocator) appendPoolUsageForStatefulSet(ipPool *v1alpha2.IPPool, newPool.Status.IPAddresses = append(newPool.Status.IPAddresses, usageEntry) } + a.updateUsage(newPool, 3) _, err := a.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), newPool, metav1.UpdateOptions{}) if err != nil { klog.Warningf("IP Pool %s update with status %+v failed: %+v", newPool.Name, newPool.Status, err) @@ -239,6 +242,7 @@ func (a *IPPoolAllocator) removeIPAddressState(ipPool *v1alpha2.IPPool, ip net.I } newPool.Status.IPAddresses = newList + a.updateUsage(newPool, 4) _, err := a.crdClient.CrdV1alpha2().IPPools().UpdateStatus(context.TODO(), newPool, metav1.UpdateOptions{}) if err != nil { @@ -622,3 +626,8 @@ func (a IPPoolAllocator) Total() int { } return allocators.Total() } + +func (a *IPPoolAllocator) updateUsage(ipPool *v1alpha2.IPPool, n int) { + ipPool.Status.Usage.Total = a.Total() + ipPool.Status.Usage.Used = len(ipPool.Status.IPAddresses) +} diff --git a/pkg/ipam/poolallocator/allocator_test.go b/pkg/ipam/poolallocator/allocator_test.go index 62725442756..44fa137b690 100644 --- a/pkg/ipam/poolallocator/allocator_test.go +++ b/pkg/ipam/poolallocator/allocator_test.go @@ -113,6 +113,7 @@ func TestAllocateIP(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 21, allocator.Total()) // Allocate specific IP from the range returnInfo, err := allocator.AllocateIP(net.ParseIP("10.2.2.101"), crdv1a2.IPAddressPhaseAllocated, fakePodOwner) @@ -135,7 +136,36 @@ func TestAllocateNext(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - poolName := uuid.New().String() + poolName := "fakePool" + ipRange := crdv1a2.IPRange{ + Start: "10.2.2.100", + End: "10.2.2.120", + } + subnetInfo := crdv1a2.SubnetInfo{ + Gateway: "10.2.2.1", + PrefixLength: 24, + } + subnetRange := crdv1a2.SubnetIPRange{IPRange: ipRange, + SubnetInfo: subnetInfo} + + pool := crdv1a2.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: poolName}, + Spec: crdv1a2.IPPoolSpec{IPRanges: []crdv1a2.SubnetIPRange{subnetRange}}, + } + + allocator := newTestIPPoolAllocator(&pool, stopCh) + assert.Equal(t, 21, allocator.Total()) + + validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101"}) +} + +// This test verifies correct behavior in case of update conflict. Allocation should be retiried +// Taking into account the latest status +func TestAllocateConflict(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + poolName := "fakePool" ipRange := crdv1a2.IPRange{ Start: "10.2.2.100", End: "10.2.2.120", @@ -185,6 +215,7 @@ func TestAllocateNextMultiRange(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 16, allocator.Total()) // Allocate the 2 available IPs from first range then switch to second range validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101", "10.2.2.2", "10.2.2.3"}) @@ -220,6 +251,7 @@ func TestAllocateNextMultiRangeExausted(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 3, allocator.Total()) // Allocate all available IPs validateAllocationSequence(t, allocator, subnetInfo, []string{"10.2.2.100", "10.2.2.101", "10.2.2.200"}) @@ -324,6 +356,7 @@ func TestReleaseResource(t *testing.T) { allocator := newTestIPPoolAllocator(&pool, stopCh) require.NotNil(t, allocator) + assert.Equal(t, 15, allocator.Total()) // Allocate the single available IPs from first range then 3 IPs from second range validateAllocationSequence(t, allocator, subnetInfo, []string{"2001::1000", "2001::2", "2001::3", "2001::4", "2001::5"})